devlyn-cli 3.2.1 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (423) hide show
  1. package/AGENTS.md +2 -2
  2. package/CLAUDE.md +2 -2
  3. package/README.md +12 -102
  4. package/bin/devlyn.js +13 -225
  5. package/bin/instruction-templates.json +6 -0
  6. package/config/skills/_shared/adapters/grok.md +1 -2
  7. package/config/skills/_shared/archive_run.py +15 -3
  8. package/config/skills/_shared/judge-role-evidence.py +21 -1
  9. package/config/skills/_shared/runtime-principles.md +1 -1
  10. package/config/skills/_shared/spec-verify-check.py +0 -653
  11. package/config/skills/_shared/state-phase-write.py +351 -25
  12. package/config/skills/_shared/task-complete.py +635 -143
  13. package/config/skills/_shared/terminal-claim-check.py +229 -4
  14. package/config/skills/_shared/verify-merge-findings.py +93 -2
  15. package/config/skills/devlyn:ideate/SKILL.md +3 -4
  16. package/config/skills/devlyn:ideate/references/elicitation.md +2 -35
  17. package/config/skills/devlyn:ideate/references/from-spec-mode.md +3 -18
  18. package/config/skills/devlyn:ideate/references/project-mode.md +1 -16
  19. package/config/skills/devlyn:ideate/references/spec-template.md +0 -2
  20. package/config/skills/devlyn:queue/SKILL.md +6 -4
  21. package/config/skills/devlyn:resolve/SKILL.md +16 -25
  22. package/config/skills/devlyn:resolve/references/free-form-mode.md +0 -14
  23. package/config/skills/devlyn:resolve/references/outer-loop.md +1 -1
  24. package/config/skills/devlyn:resolve/references/phases/build-gate.md +1 -1
  25. package/config/skills/devlyn:resolve/references/phases/cleanup.md +3 -3
  26. package/config/skills/devlyn:resolve/references/phases/implement.md +1 -1
  27. package/config/skills/devlyn:resolve/references/phases/probe-derive.md +1 -16
  28. package/config/skills/devlyn:resolve/references/phases/verify.md +4 -9
  29. package/config/skills/devlyn:resolve/references/state-schema.md +9 -7
  30. package/config/skills/devlyn:resolve/references/task-completion.md +46 -37
  31. package/{config/skills → optional-skills}/code-health-standards/SKILL.md +4 -13
  32. package/{config/skills → optional-skills}/code-review-standards/SKILL.md +4 -12
  33. package/{config/skills → optional-skills}/root-cause-analysis/SKILL.md +4 -12
  34. package/{config/skills → optional-skills}/ui-implementation-standards/SKILL.md +4 -14
  35. package/package.json +1 -57
  36. package/benchmark/auto-resolve/BENCHMARK-DESIGN.md +0 -289
  37. package/benchmark/auto-resolve/BENCHMARK-RESULTS.md +0 -341
  38. package/benchmark/auto-resolve/README.md +0 -696
  39. package/benchmark/auto-resolve/RUBRIC.md +0 -177
  40. package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/NOTES.md +0 -34
  41. package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/expected.json +0 -68
  42. package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/metadata.json +0 -10
  43. package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/setup.sh +0 -4
  44. package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/spec.md +0 -44
  45. package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/task.ko.txt +0 -8
  46. package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/task.txt +0 -8
  47. package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/NOTES.md +0 -68
  48. package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/expected.json +0 -60
  49. package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/metadata.json +0 -10
  50. package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/setup.sh +0 -17
  51. package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/spec.md +0 -51
  52. package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/task.txt +0 -9
  53. package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/verifiers/invalid.js +0 -29
  54. package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/verifiers/parallel.js +0 -50
  55. package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/NOTES.md +0 -75
  56. package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/expected.json +0 -52
  57. package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/metadata.json +0 -10
  58. package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/setup.sh +0 -171
  59. package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/spec.md +0 -50
  60. package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/task.txt +0 -9
  61. package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/NOTES.md +0 -89
  62. package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/expected.json +0 -74
  63. package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/metadata.json +0 -10
  64. package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/setup.sh +0 -251
  65. package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/spec.md +0 -57
  66. package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/task.txt +0 -13
  67. package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/verifiers/replay-malformed-body.js +0 -64
  68. package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/NOTES.md +0 -104
  69. package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/expected.json +0 -46
  70. package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/metadata.json +0 -10
  71. package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/setup.sh +0 -336
  72. package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/spec.md +0 -51
  73. package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/task.txt +0 -9
  74. package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/NOTES.md +0 -38
  75. package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/expected.json +0 -64
  76. package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/metadata.json +0 -10
  77. package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/setup.sh +0 -32
  78. package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/spec.md +0 -63
  79. package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/task.txt +0 -7
  80. package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/verifiers/exact-success.js +0 -54
  81. package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/verifiers/no-hardcoded-pricing.js +0 -47
  82. package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/verifiers/stock-error.js +0 -45
  83. package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/NOTES.md +0 -57
  84. package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/expected-pair-plan-registry.json +0 -170
  85. package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/expected.json +0 -84
  86. package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/metadata.json +0 -21
  87. package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/pair-plan.sample-fail.json +0 -214
  88. package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/pair-plan.sample-pass.json +0 -223
  89. package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/setup.sh +0 -5
  90. package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/spec.md +0 -55
  91. package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/task.txt +0 -14
  92. package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/NOTES.md +0 -39
  93. package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/expected.json +0 -62
  94. package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/metadata.json +0 -10
  95. package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/setup.sh +0 -2
  96. package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/spec.md +0 -67
  97. package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/task.txt +0 -7
  98. package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/verifiers/error-order.js +0 -55
  99. package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/verifiers/priority-blocked.js +0 -48
  100. package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/NOTES.md +0 -35
  101. package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/expected.json +0 -56
  102. package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/metadata.json +0 -10
  103. package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/setup.sh +0 -2
  104. package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/spec.md +0 -64
  105. package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/task.txt +0 -7
  106. package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/verifiers/conflicting-duplicate.js +0 -34
  107. package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/verifiers/idempotent-close.js +0 -41
  108. package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/NOTES.md +0 -39
  109. package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/expected.json +0 -56
  110. package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/metadata.json +0 -10
  111. package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/setup.sh +0 -2
  112. package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/spec.md +0 -76
  113. package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/task.txt +0 -7
  114. package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/verifiers/priority-rollback.js +0 -64
  115. package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/verifiers/single-warehouse-fefo.js +0 -66
  116. package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/NOTES.md +0 -40
  117. package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/expected.json +0 -66
  118. package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/metadata.json +0 -10
  119. package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/setup.sh +0 -36
  120. package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/spec.md +0 -71
  121. package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/task.txt +0 -7
  122. package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/verifiers/catalog-source.js +0 -57
  123. package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/verifiers/exact-success.js +0 -63
  124. package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/verifiers/stock-error.js +0 -34
  125. package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/NOTES.md +0 -31
  126. package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/expected.json +0 -68
  127. package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/metadata.json +0 -10
  128. package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/setup.sh +0 -17
  129. package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/spec.md +0 -68
  130. package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/task.txt +0 -7
  131. package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/verifiers/conflicting-duplicate.js +0 -29
  132. package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/verifiers/exact-payout.js +0 -58
  133. package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/verifiers/rules-source.js +0 -56
  134. package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/NOTES.md +0 -35
  135. package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/expected-pair-plan-registry.json +0 -162
  136. package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/expected.json +0 -67
  137. package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/metadata.json +0 -19
  138. package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/setup.sh +0 -4
  139. package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/spec.md +0 -55
  140. package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/task.txt +0 -9
  141. package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/NOTES.md +0 -34
  142. package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/expected.json +0 -57
  143. package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/metadata.json +0 -10
  144. package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/setup.sh +0 -2
  145. package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/spec.md +0 -67
  146. package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/task.txt +0 -7
  147. package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/verifiers/duplicate-event-error.js +0 -35
  148. package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/verifiers/priority-transfer-rollback.js +0 -53
  149. package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/NOTES.md +0 -38
  150. package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/expected.json +0 -57
  151. package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/metadata.json +0 -10
  152. package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/setup.sh +0 -2
  153. package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/spec.md +0 -70
  154. package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/task.txt +0 -3
  155. package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/verifiers/duplicate-renewal-error.js +0 -42
  156. package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/verifiers/priority-credit-rollback.js +0 -70
  157. package/benchmark/auto-resolve/fixtures/F4-web-browser-design/NOTES.md +0 -47
  158. package/benchmark/auto-resolve/fixtures/F4-web-browser-design/expected.json +0 -57
  159. package/benchmark/auto-resolve/fixtures/F4-web-browser-design/metadata.json +0 -10
  160. package/benchmark/auto-resolve/fixtures/F4-web-browser-design/setup.sh +0 -6
  161. package/benchmark/auto-resolve/fixtures/F4-web-browser-design/spec.md +0 -48
  162. package/benchmark/auto-resolve/fixtures/F4-web-browser-design/task.txt +0 -9
  163. package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/NOTES.md +0 -45
  164. package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/expected.json +0 -65
  165. package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/metadata.json +0 -10
  166. package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/setup.sh +0 -55
  167. package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/spec.md +0 -48
  168. package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/task.txt +0 -7
  169. package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/NOTES.md +0 -43
  170. package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/expected.json +0 -77
  171. package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/metadata.json +0 -10
  172. package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/setup.sh +0 -4
  173. package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/spec.md +0 -48
  174. package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/task.txt +0 -10
  175. package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/NOTES.md +0 -57
  176. package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/expected.json +0 -76
  177. package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/metadata.json +0 -10
  178. package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/setup.sh +0 -36
  179. package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/spec.md +0 -45
  180. package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/task.txt +0 -7
  181. package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/NOTES.md +0 -53
  182. package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/expected.json +0 -63
  183. package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/metadata.json +0 -10
  184. package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/setup.sh +0 -4
  185. package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/spec.md +0 -48
  186. package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/task.txt +0 -1
  187. package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/NOTES.md +0 -105
  188. package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/expected.json +0 -74
  189. package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/metadata.json +0 -10
  190. package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/setup.sh +0 -28
  191. package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/spec.md +0 -59
  192. package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/task.txt +0 -5
  193. package/benchmark/auto-resolve/fixtures/SCHEMA.md +0 -188
  194. package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/NOTES.md +0 -37
  195. package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/RETIRED.md +0 -13
  196. package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/expected.json +0 -56
  197. package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/metadata.json +0 -10
  198. package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/setup.sh +0 -18
  199. package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/spec.md +0 -69
  200. package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/task.txt +0 -7
  201. package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/verifiers/exact-proration.js +0 -48
  202. package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/verifiers/rules-source-and-conflict.js +0 -79
  203. package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/NOTES.md +0 -54
  204. package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/RETIRED.md +0 -7
  205. package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/expected.json +0 -67
  206. package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/metadata.json +0 -10
  207. package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/setup.sh +0 -2
  208. package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/spec.md +0 -67
  209. package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/task.txt +0 -5
  210. package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/verifiers/policy-precedence.js +0 -72
  211. package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/verifiers/validation-and-immutability.js +0 -43
  212. package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/verifiers/validation-boundary.js +0 -116
  213. package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/NOTES.md +0 -35
  214. package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/RETIRED.md +0 -12
  215. package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/expected.json +0 -58
  216. package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/metadata.json +0 -10
  217. package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/setup.sh +0 -2
  218. package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/spec.md +0 -73
  219. package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/task.txt +0 -17
  220. package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/verifiers/mixed-idempotent-settlement.js +0 -53
  221. package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/verifiers/rejection-boundaries.js +0 -74
  222. package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/NOTES.md +0 -28
  223. package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/RETIRED.md +0 -15
  224. package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/expected.json +0 -57
  225. package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/metadata.json +0 -10
  226. package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/setup.sh +0 -50
  227. package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/spec.md +0 -59
  228. package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/task.txt +0 -9
  229. package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/verifiers/ordered-settlement.js +0 -88
  230. package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/verifiers/reject-invalid.js +0 -45
  231. package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/NOTES.md +0 -30
  232. package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/RETIRED.md +0 -16
  233. package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/expected.json +0 -64
  234. package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/metadata.json +0 -10
  235. package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/setup.sh +0 -21
  236. package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/spec.md +0 -58
  237. package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/task.txt +0 -9
  238. package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/verifiers/dup-op.js +0 -54
  239. package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/verifiers/priority-apply.js +0 -82
  240. package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/verifiers/rollback.js +0 -77
  241. package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/NOTES.md +0 -99
  242. package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/RETIRED.md +0 -18
  243. package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/expected.json +0 -64
  244. package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/metadata.json +0 -10
  245. package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/setup.sh +0 -4
  246. package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/spec.md +0 -61
  247. package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/task.txt +0 -9
  248. package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/verifiers/correctness-small.js +0 -135
  249. package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/verifiers/scale.js +0 -100
  250. package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/verifiers/validation.js +0 -75
  251. package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/NOTES.md +0 -93
  252. package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/RETIRED.md +0 -16
  253. package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/expected.json +0 -65
  254. package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/metadata.json +0 -10
  255. package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/setup.sh +0 -18
  256. package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/spec.md +0 -66
  257. package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/task.txt +0 -9
  258. package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/verifiers/correctness-small.js +0 -84
  259. package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/verifiers/scale.js +0 -184
  260. package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/verifiers/validation.js +0 -74
  261. package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/NOTES.md +0 -60
  262. package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/RETIRED.md +0 -29
  263. package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/expected.json +0 -73
  264. package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/metadata.json +0 -10
  265. package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/setup.sh +0 -28
  266. package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/spec.md +0 -58
  267. package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/task.txt +0 -5
  268. package/benchmark/auto-resolve/fixtures/test-repo/README.md +0 -12
  269. package/benchmark/auto-resolve/fixtures/test-repo/bin/cli.js +0 -63
  270. package/benchmark/auto-resolve/fixtures/test-repo/package-lock.json +0 -823
  271. package/benchmark/auto-resolve/fixtures/test-repo/package.json +0 -22
  272. package/benchmark/auto-resolve/fixtures/test-repo/playwright.config.js +0 -17
  273. package/benchmark/auto-resolve/fixtures/test-repo/server/index.js +0 -37
  274. package/benchmark/auto-resolve/fixtures/test-repo/tests/cli.test.js +0 -25
  275. package/benchmark/auto-resolve/fixtures/test-repo/tests/server.test.js +0 -58
  276. package/benchmark/auto-resolve/fixtures/test-repo/web/index.html +0 -37
  277. package/benchmark/auto-resolve/results/20260507-f10-f11-tier1-full-pipeline/headroom-gate.json +0 -25
  278. package/benchmark/auto-resolve/results/20260507-f10-f11-tier1-full-pipeline/headroom-gate.md +0 -10
  279. package/benchmark/auto-resolve/results/20260508-f22-exact-error-headroom/headroom-gate.json +0 -18
  280. package/benchmark/auto-resolve/results/20260508-f22-exact-error-headroom/headroom-gate.md +0 -9
  281. package/benchmark/auto-resolve/results/20260508-f26-headroom/headroom-gate.json +0 -18
  282. package/benchmark/auto-resolve/results/20260508-f26-headroom/headroom-gate.md +0 -9
  283. package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F16-cli-quote-tax-rules/l2_risk_probes/result.json +0 -28
  284. package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F23-cli-fulfillment-wave/l2_risk_probes/result.json +0 -28
  285. package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F25-cli-cart-promotion-rules/l2_risk_probes/result.json +0 -30
  286. package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.json +0 -82
  287. package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.md +0 -18
  288. package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.json +0 -46
  289. package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.md +0 -17
  290. package/benchmark/auto-resolve/results/20260511-f12-webhook-headroom/headroom-gate.json +0 -28
  291. package/benchmark/auto-resolve/results/20260511-f12-webhook-headroom/headroom-gate.md +0 -15
  292. package/benchmark/auto-resolve/results/20260511-f15-concurrency-headroom/headroom-gate.json +0 -28
  293. package/benchmark/auto-resolve/results/20260511-f15-concurrency-headroom/headroom-gate.md +0 -15
  294. package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/F21-cli-scheduler-priority/l2_risk_probes/result.json +0 -32
  295. package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.json +0 -44
  296. package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.md +0 -16
  297. package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.json +0 -18
  298. package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.md +0 -9
  299. package/benchmark/auto-resolve/results/20260511-f3-http-error-headroom/headroom-gate.json +0 -28
  300. package/benchmark/auto-resolve/results/20260511-f3-http-error-headroom/headroom-gate.md +0 -15
  301. package/benchmark/auto-resolve/results/20260512-f2-medium-headroom/headroom-gate.json +0 -28
  302. package/benchmark/auto-resolve/results/20260512-f2-medium-headroom/headroom-gate.md +0 -15
  303. package/benchmark/auto-resolve/results/20260512-f31-seat-rebalance-headroom/headroom-gate.json +0 -28
  304. package/benchmark/auto-resolve/results/20260512-f31-seat-rebalance-headroom/headroom-gate.md +0 -15
  305. package/benchmark/auto-resolve/results/20260512-f32-subscription-renewal-headroom/headroom-gate.json +0 -28
  306. package/benchmark/auto-resolve/results/20260512-f32-subscription-renewal-headroom/headroom-gate.md +0 -15
  307. package/benchmark/auto-resolve/results/20260512-f4-web-headroom/headroom-gate.json +0 -28
  308. package/benchmark/auto-resolve/results/20260512-f4-web-headroom/headroom-gate.md +0 -15
  309. package/benchmark/auto-resolve/results/20260512-f5-fixloop-headroom/headroom-gate.json +0 -28
  310. package/benchmark/auto-resolve/results/20260512-f5-fixloop-headroom/headroom-gate.md +0 -15
  311. package/benchmark/auto-resolve/results/20260512-f6-checksum-headroom/headroom-gate.json +0 -28
  312. package/benchmark/auto-resolve/results/20260512-f6-checksum-headroom/headroom-gate.md +0 -15
  313. package/benchmark/auto-resolve/results/20260512-f7-scope-headroom/headroom-gate.json +0 -28
  314. package/benchmark/auto-resolve/results/20260512-f7-scope-headroom/headroom-gate.md +0 -15
  315. package/benchmark/auto-resolve/results/20260512-f9-e2e-headroom/headroom-gate.json +0 -28
  316. package/benchmark/auto-resolve/results/20260512-f9-e2e-headroom/headroom-gate.md +0 -15
  317. package/benchmark/auto-resolve/run-real-benchmark.md +0 -303
  318. package/benchmark/auto-resolve/scripts/audit-headroom-rejections.py +0 -441
  319. package/benchmark/auto-resolve/scripts/audit-pair-evidence.py +0 -1256
  320. package/benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py +0 -306
  321. package/benchmark/auto-resolve/scripts/check-f9-artifacts.py +0 -268
  322. package/benchmark/auto-resolve/scripts/collect-swebench-predictions.py +0 -108
  323. package/benchmark/auto-resolve/scripts/compile-report.py +0 -493
  324. package/benchmark/auto-resolve/scripts/fetch-swebench-instances.py +0 -129
  325. package/benchmark/auto-resolve/scripts/frozen-verify-gate.py +0 -434
  326. package/benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py +0 -612
  327. package/benchmark/auto-resolve/scripts/headroom-gate.py +0 -378
  328. package/benchmark/auto-resolve/scripts/iter-0033c-compare.py +0 -683
  329. package/benchmark/auto-resolve/scripts/iter-0033c-l1-summary.py +0 -97
  330. package/benchmark/auto-resolve/scripts/judge-opus-pass.sh +0 -542
  331. package/benchmark/auto-resolve/scripts/judge.sh +0 -565
  332. package/benchmark/auto-resolve/scripts/oracle-scope-tier-a.py +0 -256
  333. package/benchmark/auto-resolve/scripts/oracle-scope-tier-b.py +0 -287
  334. package/benchmark/auto-resolve/scripts/oracle-test-fidelity.py +0 -328
  335. package/benchmark/auto-resolve/scripts/pair-candidate-frontier.py +0 -469
  336. package/benchmark/auto-resolve/scripts/pair-plan-idgen.py +0 -401
  337. package/benchmark/auto-resolve/scripts/pair-plan-lint.py +0 -475
  338. package/benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh +0 -103
  339. package/benchmark/auto-resolve/scripts/pair_evidence_contract.py +0 -270
  340. package/benchmark/auto-resolve/scripts/prepare-swebench-frozen-case.py +0 -273
  341. package/benchmark/auto-resolve/scripts/prepare-swebench-frozen-corpus.py +0 -148
  342. package/benchmark/auto-resolve/scripts/prepare-swebench-solver-worktree.py +0 -210
  343. package/benchmark/auto-resolve/scripts/recent-benchmark-summary.py +0 -232
  344. package/benchmark/auto-resolve/scripts/run-fixture.sh +0 -1024
  345. package/benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh +0 -683
  346. package/benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh +0 -458
  347. package/benchmark/auto-resolve/scripts/run-headroom-candidate.sh +0 -336
  348. package/benchmark/auto-resolve/scripts/run-iter-0033c.sh +0 -208
  349. package/benchmark/auto-resolve/scripts/run-suite.sh +0 -251
  350. package/benchmark/auto-resolve/scripts/run-swebench-frozen-corpus.sh +0 -310
  351. package/benchmark/auto-resolve/scripts/run-swebench-solver-batch.sh +0 -257
  352. package/benchmark/auto-resolve/scripts/ship-gate.py +0 -391
  353. package/benchmark/auto-resolve/scripts/solo-ceiling-avoidance.py +0 -53
  354. package/benchmark/auto-resolve/scripts/solo-headroom-hypothesis.py +0 -77
  355. package/benchmark/auto-resolve/scripts/swebench-frozen-matrix.py +0 -478
  356. package/benchmark/auto-resolve/scripts/test-audit-headroom-rejections.sh +0 -288
  357. package/benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh +0 -1672
  358. package/benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh +0 -1015
  359. package/benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh +0 -491
  360. package/benchmark/auto-resolve/scripts/test-check-f9-artifacts.sh +0 -91
  361. package/benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh +0 -517
  362. package/benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh +0 -610
  363. package/benchmark/auto-resolve/scripts/test-headroom-gate.sh +0 -401
  364. package/benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh +0 -525
  365. package/benchmark/auto-resolve/scripts/test-iter-0033c-l1-summary.sh +0 -254
  366. package/benchmark/auto-resolve/scripts/test-lint-fixtures.sh +0 -580
  367. package/benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh +0 -591
  368. package/benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh +0 -497
  369. package/benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh +0 -401
  370. package/benchmark/auto-resolve/scripts/test-run-swebench-solver-batch.sh +0 -111
  371. package/benchmark/auto-resolve/scripts/test-ship-gate.sh +0 -1189
  372. package/benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh +0 -1221
  373. package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/NOTES.md +0 -28
  374. package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/expected.json +0 -63
  375. package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/metadata.json +0 -10
  376. package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/setup.sh +0 -3
  377. package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/spec.md +0 -47
  378. package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/task.txt +0 -1
  379. package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/NOTES.md +0 -34
  380. package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/expected.json +0 -53
  381. package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/metadata.json +0 -10
  382. package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/setup.sh +0 -3
  383. package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/spec.md +0 -50
  384. package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/task.txt +0 -1
  385. package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/verifiers/duplicate-order-error.js +0 -27
  386. package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/verifiers/priority-stock-reservation.js +0 -44
  387. package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/NOTES.md +0 -34
  388. package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/expected.json +0 -55
  389. package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/metadata.json +0 -10
  390. package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/setup.sh +0 -3
  391. package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/spec.md +0 -52
  392. package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/task.txt +0 -1
  393. package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/verifiers/duplicate-ticket-error.js +0 -29
  394. package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/verifiers/priority-agent-assignment.js +0 -48
  395. package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/NOTES.md +0 -34
  396. package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/expected.json +0 -55
  397. package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/metadata.json +0 -10
  398. package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/setup.sh +0 -3
  399. package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/spec.md +0 -55
  400. package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/task.txt +0 -1
  401. package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/verifiers/duplicate-return-error.js +0 -43
  402. package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/verifiers/priority-return-routing.js +0 -70
  403. package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/NOTES.md +0 -37
  404. package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/expected.json +0 -54
  405. package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/metadata.json +0 -10
  406. package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/setup.sh +0 -3
  407. package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/spec.md +0 -59
  408. package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/task.txt +0 -1
  409. package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/verifiers/credit-ledger-priority.js +0 -98
  410. package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/verifiers/duplicate-charge-error.js +0 -38
  411. package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/NOTES.md +0 -36
  412. package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/expected.json +0 -56
  413. package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/metadata.json +0 -10
  414. package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/setup.sh +0 -3
  415. package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/spec.md +0 -59
  416. package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/task.txt +0 -1
  417. package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/verifiers/duplicate-refund-error.js +0 -41
  418. package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/verifiers/priority-refund-ledger.js +0 -65
  419. package/config/skills/_shared/pair-plan-schema.md +0 -298
  420. package/scripts/lint-fixtures.sh +0 -352
  421. package/scripts/lint-shadow-fixtures.sh +0 -58
  422. package/scripts/lint-skills.sh +0 -4265
  423. package/scripts/test-owner-phases.py +0 -238
@@ -1,4265 +0,0 @@
1
- #!/usr/bin/env bash
2
- # lint-skills.sh — structural quality checks for the devlyn harness.
3
- #
4
- # Gates the three things that have drifted in the past:
5
- # 1. Forbidden MCP / stale-model references in skills, README, installer.
6
- # 2. Missing `name:` in skill frontmatter (Anthropic spec violation).
7
- # 3. Source ↔ installed mirror drift on the harness critical path.
8
- #
9
- # Exit 0 = clean. Non-zero = fails; prints offending file:line per check.
10
-
11
- set -u
12
- cd "$(dirname "$0")/.."
13
-
14
- red=$(printf '\033[31m'); green=$(printf '\033[32m'); dim=$(printf '\033[2m'); reset=$(printf '\033[0m')
15
- fail=0
16
-
17
- section() { printf '\n%s=== %s ===%s\n' "$dim" "$1" "$reset"; }
18
- ok() { printf ' %s✓%s %s\n' "$green" "$reset" "$1"; }
19
- bad() { printf ' %s✗%s %s\n' "$red" "$reset" "$1"; fail=1; }
20
-
21
- make_temp_file() {
22
- local __var="$1"
23
- shift || true
24
- local path
25
- if ! path=$(command mktemp "$@"); then
26
- bad "mktemp failed: ${*:-<default>}"
27
- return 1
28
- fi
29
- printf -v "$__var" '%s' "$path"
30
- }
31
-
32
- make_temp_dir() {
33
- local __var="$1"
34
- shift || true
35
- local path
36
- if ! path=$(command mktemp -d "$@"); then
37
- bad "mktemp -d failed: ${*:-<default>}"
38
- return 1
39
- fi
40
- printf -v "$__var" '%s' "$path"
41
- }
42
-
43
- section "Check 0a: Temp allocation fails closed"
44
- direct_mktemp=$(grep -nE '(^|[ =])mktemp( |$)|\$\([[:space:]]*mktemp' scripts/lint-skills.sh \
45
- | grep -v 'command mktemp' \
46
- | grep -v 'make_temp_' \
47
- | grep -v 'direct_mktemp=' || true)
48
- if [ -z "$direct_mktemp" ]; then
49
- ok "lint-skills.sh uses temp allocation helpers instead of direct mktemp"
50
- else
51
- while IFS= read -r f; do bad "$f"; done <<< "$direct_mktemp"
52
- fi
53
-
54
- # iter-0034 Phase 4 cutover (2026-05-03): legacy skill paths dropped.
55
- # Surface is the 2-skill product (`/devlyn:resolve` + `/devlyn:ideate`)
56
- # plus the `_shared/` kernel. Keep this list single-source so all installed
57
- # mirror parity checks cover the same files.
58
- critical_path_files=$(cat <<'EOF'
59
- _shared/.ruff.toml
60
- _shared/process-evidence.py
61
- _shared/platform-support.py
62
- _shared/invocation-receipt.py
63
- _shared/role-config.py
64
- _shared/judge-role-evidence.py
65
- _shared/spec-verify-check.py
66
- _shared/judge-output-parser.py
67
- _shared/collect-codex-findings.py
68
- _shared/verify-merge-findings.py
69
- _shared/archive_run.py
70
- _shared/task-complete.py
71
- _shared/state-phase-write.py
72
- _shared/terminal-claim-check.py
73
- _shared/resolve-stop-hook.py
74
- _shared/resolve-bootstrap.py
75
- devlyn:ideate/SKILL.md
76
- devlyn:ideate/references/spec-template.md
77
- devlyn:ideate/references/elicitation.md
78
- devlyn:ideate/references/project-mode.md
79
- devlyn:ideate/references/from-spec-mode.md
80
- devlyn:resolve/SKILL.md
81
- devlyn:queue/SKILL.md
82
- devlyn:engines/SKILL.md
83
- devlyn:resolve/references/state-schema.md
84
- devlyn:resolve/references/task-completion.md
85
- devlyn:resolve/references/outer-loop.md
86
- devlyn:resolve/references/free-form-mode.md
87
- devlyn:resolve/references/phases/plan.md
88
- devlyn:resolve/references/phases/probe-derive.md
89
- devlyn:resolve/references/phases/implement.md
90
- devlyn:resolve/references/phases/surface-close.md
91
- devlyn:resolve/references/phases/build-gate.md
92
- devlyn:resolve/references/phases/cleanup.md
93
- devlyn:resolve/references/phases/verify.md
94
- _shared/expected.schema.json
95
- _shared/adapters/README.md
96
- _shared/adapters/claude.md
97
- _shared/adapters/codex.md
98
- _shared/adapters/grok.md
99
- _shared/adapters/omp.md
100
- _shared/codex-config.md
101
- _shared/codex-monitored.sh
102
- _shared/engine-preflight.md
103
- _shared/pair-plan-schema.md
104
- _shared/runtime-principles.md
105
- EOF
106
- )
107
-
108
- check_skill_mirror_parity() {
109
- local target_dir="$1"
110
- local skip_msg="$2"
111
- local missing_prefix="$3"
112
- local differ_suffix="$4"
113
- local ok_msg="$5"
114
- local drift=0
115
- local rel src dst
116
-
117
- if [ ! -d "$target_dir" ]; then
118
- ok "$skip_msg"
119
- return
120
- fi
121
-
122
- while IFS= read -r rel; do
123
- [ -n "$rel" ] || continue
124
- src="config/skills/$rel"
125
- dst="$target_dir/$rel"
126
- if [ ! -f "$src" ] || [ ! -f "$dst" ]; then
127
- bad "$missing_prefix: $rel"; drift=1; continue
128
- fi
129
- if ! diff -q "$src" "$dst" >/dev/null 2>&1; then
130
- bad "$rel — $differ_suffix"
131
- drift=1
132
- fi
133
- done <<< "$critical_path_files"
134
-
135
- # iter-0009: codex-monitored.sh must be executable in installed mirrors
136
- # (skills trees get copied into work dirs for variant arms; bash refuses to
137
- # run a non-executable wrapper).
138
- if [ -f "$target_dir/_shared/codex-monitored.sh" ] \
139
- && [ ! -x "$target_dir/_shared/codex-monitored.sh" ]; then
140
- bad "_shared/codex-monitored.sh — not executable in ${target_dir} mirror"
141
- drift=1
142
- fi
143
- if [ $drift -eq 0 ]; then
144
- ok "$ok_msg"
145
- fi
146
- }
147
-
148
- # ---------------------------------------------------------------------------
149
- # 1. No MCP references in managed source or user-facing docs.
150
- # ---------------------------------------------------------------------------
151
- section "Check 1: No mcp__codex-cli__ outside _shared / archive"
152
- # Legal places: config/skills/_shared/codex-config.md (explicitly says "MCP is not used"),
153
- # archival snapshots, and tests.
154
- offenders=$(git grep -Il -- 'mcp__codex-cli__' -- \
155
- config/skills \
156
- benchmark \
157
- README.md \
158
- CLAUDE.md \
159
- bin/ \
160
- ':!config/skills/_shared/codex-config.md' \
161
- ':!config/skills/roadmap-archival-workspace/**' \
162
- ':!config/skills/devlyn:auto-resolve-workspace/**' \
163
- ':!config/skills/devlyn:ideate-workspace/**' \
164
- ':!config/skills/preflight-workspace/**' \
165
- ':!benchmark/auto-resolve/external/**' \
166
- ':!benchmark/auto-resolve/results/**' \
167
- ':!benchmark/ceiling/results/**' \
168
- ':!benchmark/auto-resolve/PILOT-RESULTS*' \
169
- 2>/dev/null || true)
170
- if [ -z "$offenders" ]; then
171
- ok "no MCP references in managed files"
172
- else
173
- while IFS= read -r f; do bad "$f"; done <<< "$offenders"
174
- fi
175
-
176
- # ---------------------------------------------------------------------------
177
- # 2. No "Requires Codex MCP" prose.
178
- # ---------------------------------------------------------------------------
179
- section "Check 2: No 'Requires Codex MCP' prose"
180
- offenders=$(git grep -Il -- 'Requires Codex MCP\|Codex MCP server\|Codex MCP available\|Codex MCP disconnected' -- \
181
- config/skills \
182
- benchmark \
183
- README.md \
184
- CLAUDE.md \
185
- bin/ \
186
- ':!config/skills/roadmap-archival-workspace/**' \
187
- ':!config/skills/devlyn:auto-resolve-workspace/**' \
188
- ':!config/skills/devlyn:ideate-workspace/**' \
189
- ':!config/skills/preflight-workspace/**' \
190
- ':!benchmark/auto-resolve/external/**' \
191
- ':!benchmark/auto-resolve/results/**' \
192
- ':!benchmark/ceiling/results/**' \
193
- ':!benchmark/auto-resolve/PILOT-RESULTS*' \
194
- 2>/dev/null || true)
195
- if [ -z "$offenders" ]; then
196
- ok "no Codex MCP prose"
197
- else
198
- while IFS= read -r f; do bad "$f"; done <<< "$offenders"
199
- fi
200
-
201
- # ---------------------------------------------------------------------------
202
- # 3. No stale model strings (gpt-5.0..5.4 hardcoded outside config).
203
- # ---------------------------------------------------------------------------
204
- section "Check 3: No hardcoded pre-5.5 model strings"
205
- offenders=$(grep -RInE 'gpt-5\.[0-4][^.]' \
206
- config/skills CLAUDE.md README.md 2>/dev/null \
207
- | grep -v 'config/skills/_shared/codex-config.md' \
208
- | grep -v 'config/skills/roadmap-archival-workspace/' \
209
- | grep -v 'config/skills/devlyn:auto-resolve-workspace/' \
210
- | grep -v 'config/skills/devlyn:ideate-workspace/' \
211
- | grep -v 'config/skills/preflight-workspace/' \
212
- | grep -v 'evals\.json' \
213
- || true)
214
- if [ -z "$offenders" ]; then
215
- ok "no hardcoded pre-5.5 strings"
216
- else
217
- while IFS= read -r f; do bad "$f"; done <<< "$offenders"
218
- fi
219
-
220
- # ---------------------------------------------------------------------------
221
- # 4. No model-pinned Claude references, any generation, except the
222
- # validated explicit-effort capability declaration.
223
- # ---------------------------------------------------------------------------
224
- section "Check 4: No model-pinned Claude references"
225
- offenders=$(grep -RInE 'Claude (Opus|Sonnet|Haiku|Fable)|claude-(opus|sonnet|haiku|fable)-[0-9]' \
226
- config/skills 2>/dev/null \
227
- | grep -v 'config/skills/roadmap-archival-workspace/' \
228
- | grep -v 'config/skills/devlyn:auto-resolve-workspace/' \
229
- | grep -v 'config/skills/devlyn:ideate-workspace/' \
230
- | grep -v 'config/skills/preflight-workspace/' \
231
- | grep -vE '^config/skills/_shared/adapters/claude\.md:[0-9]+:<!-- devlyn-effort 2\.1\.263 claude-fable-5-1 low,medium,high,xhigh,max -->$' \
232
- | cut -d: -f1 \
233
- | sort -u \
234
- || true)
235
- if [ -z "$offenders" ]; then
236
- ok "no model-pinned Claude references"
237
- else
238
- while IFS= read -r f; do bad "$f"; done <<< "$offenders"
239
- fi
240
-
241
- # ---------------------------------------------------------------------------
242
- # 5. Every devlyn:* skill has `name:` in frontmatter.
243
- # ---------------------------------------------------------------------------
244
- section "Check 5: devlyn:* SKILL.md has name: field"
245
- missing=0
246
- for skill in config/skills/devlyn:*/SKILL.md; do
247
- [ -f "$skill" ] || continue
248
- if ! head -20 "$skill" | grep -q '^name:'; then
249
- bad "$skill — missing 'name:' in frontmatter"
250
- missing=1
251
- fi
252
- done
253
- if [ $missing -eq 0 ]; then
254
- ok "all devlyn:* skills have name: field"
255
- fi
256
-
257
- # ---------------------------------------------------------------------------
258
- # 5a. devlyn:design-ui is a required skill, not an optional addon.
259
- # ---------------------------------------------------------------------------
260
- section "Check 5a: devlyn:design-ui is required"
261
- if [ -f "config/skills/devlyn:design-ui/SKILL.md" ]; then
262
- ok "devlyn:design-ui source lives in config/skills"
263
- else
264
- bad "devlyn:design-ui must be a required skill under config/skills"
265
- fi
266
- if [ ! -e "optional-skills/devlyn:design-ui" ]; then
267
- ok "devlyn:design-ui is not in optional-skills"
268
- else
269
- bad "devlyn:design-ui must not be installed as an optional addon"
270
- fi
271
- if grep -Fq "const DEVLYN_CORE_SKILLS = ['devlyn:resolve', 'devlyn:ideate', 'devlyn:design-ui', 'devlyn:engines', 'devlyn:queue', '_shared'];" bin/devlyn.js \
272
- && grep -Fq "skillsToInstall: DEVLYN_CORE_SKILLS" bin/devlyn.js; then
273
- ok "Codex install includes devlyn:design-ui (via shared DEVLYN_CORE_SKILLS bundle)"
274
- else
275
- bad "Codex skillsToInstall must include devlyn:design-ui (shared DEVLYN_CORE_SKILLS bundle)"
276
- fi
277
- if ! grep -F "name: 'devlyn:design-ui'" bin/devlyn.js >/dev/null 2>&1; then
278
- ok "devlyn:design-ui is absent from OPTIONAL_ADDONS"
279
- else
280
- bad "devlyn:design-ui must not be listed in OPTIONAL_ADDONS"
281
- fi
282
-
283
- # ---------------------------------------------------------------------------
284
- # 5b. Every completed managed-skills install writes its own version marker.
285
- # ---------------------------------------------------------------------------
286
- section "Check 5b: Installed skill roots carry exact version markers"
287
- if make_temp_dir tmp_install_marker /tmp/devlyn-install-marker.XXXXXX; then
288
- marker_home="$tmp_install_marker/home"
289
- marker_project="$tmp_install_marker/project"
290
- timeout_low="$tmp_install_marker/timeout-low"
291
- timeout_high="$tmp_install_marker/timeout-high"
292
- installer="$PWD/bin/devlyn.js"
293
- mkdir -p "$marker_home" "$marker_project" "$timeout_low/.claude" "$timeout_high/.claude"
294
- printf '{"env":{"BASH_MAX_TIMEOUT_MS":"600000"}}\n' > "$timeout_low/.claude/settings.json"
295
- printf '{"env":{"BASH_MAX_TIMEOUT_MS":"7200000"}}\n' > "$timeout_high/.claude/settings.json"
296
-
297
- if (cd "$marker_project" \
298
- && HOME="$marker_home" node "$installer" -y >"$tmp_install_marker/claude.log" 2>&1 \
299
- && HOME="$marker_home" node "$installer" agents all >"$tmp_install_marker/agents.log" 2>&1) \
300
- && (cd "$timeout_low" && HOME="$marker_home" node "$installer" -y >"$tmp_install_marker/timeout-low.log" 2>&1) \
301
- && cp "$timeout_low/.claude/settings.json" "$tmp_install_marker/timeout-low-first.json" \
302
- && (cd "$timeout_low" && HOME="$marker_home" node "$installer" -y >"$tmp_install_marker/timeout-low-second.log" 2>&1) \
303
- && cmp -s "$timeout_low/.claude/settings.json" "$tmp_install_marker/timeout-low-first.json" \
304
- && (cd "$timeout_high" && HOME="$marker_home" node "$installer" -y >"$tmp_install_marker/timeout-high.log" 2>&1); then
305
- if python3 - "$installer" "$marker_home" "$marker_project" "$timeout_low" "$timeout_high" <<'PY'
306
- import json
307
- import pathlib
308
- import stat
309
- import sys
310
-
311
- installer = pathlib.Path(sys.argv[1])
312
- home = pathlib.Path(sys.argv[2])
313
- project = pathlib.Path(sys.argv[3])
314
- timeout_low = pathlib.Path(sys.argv[4])
315
- timeout_high = pathlib.Path(sys.argv[5])
316
- package = json.loads((installer.parent.parent / "package.json").read_text())
317
- expected = {"schemaVersion": 1, "package": package["name"], "version": package["version"]}
318
- roots = [
319
- project / ".claude" / "skills",
320
- home / ".codex" / "skills",
321
- home / ".agents" / "skills",
322
- home / ".grok" / "skills",
323
- ]
324
- for root in roots:
325
- marker = root / ".devlyn-install.json"
326
- actual = json.loads(marker.read_text())
327
- if actual != expected:
328
- raise SystemExit(f"{marker}: expected {expected}, got {actual}")
329
- if stat.S_IMODE(marker.stat().st_mode) != 0o600:
330
- raise SystemExit(f"{marker}: expected mode 0600")
331
- if list(root.glob(".devlyn-install.json.*.tmp")):
332
- raise SystemExit(f"{root}: stale marker temp file")
333
- for target, expected_timeout in (
334
- (project, "3600000"),
335
- (timeout_low, "3600000"),
336
- (timeout_high, "7200000"),
337
- ):
338
- env = json.loads((target / ".claude" / "settings.json").read_text())["env"]
339
- if env.get("BASH_MAX_TIMEOUT_MS") != expected_timeout:
340
- raise SystemExit(f"{target}: expected BASH_MAX_TIMEOUT_MS={expected_timeout}")
341
- if "BASH_DEFAULT_TIMEOUT_MS" in env:
342
- raise SystemExit(f"{target}: BASH_DEFAULT_TIMEOUT_MS must not be installed")
343
- PY
344
- then
345
- if grep -Fxq '.claude/skills/.devlyn-install.json' "$marker_project/.gitignore"; then
346
- ok "managed roots have exact 0600 markers; Bash max is installed, raised, preserved, and idempotent"
347
- else
348
- bad "Claude project install must ignore its local marker"
349
- fi
350
- else
351
- bad "completed installs must write the exact per-root marker"
352
- fi
353
- else
354
- bad "isolated managed-skills install failed"
355
- fi
356
-
357
- incomplete="$tmp_install_marker/incomplete"
358
- mkdir -p "$incomplete/package" "$incomplete/home/.codex/skills" "$incomplete/claude/.claude/skills" "$incomplete/codex"
359
- cp -R bin config package.json AGENTS.md CLAUDE.md "$incomplete/package/"
360
- rm -rf "$incomplete/package/config/skills/devlyn:queue"
361
- printf '{"version":"stale"}\n' > "$incomplete/home/.codex/skills/.devlyn-install.json"
362
- printf '{"version":"stale"}\n' > "$incomplete/claude/.claude/skills/.devlyn-install.json"
363
- if ! (cd "$incomplete/claude" \
364
- && HOME="$incomplete/home" node "$incomplete/package/bin/devlyn.js" -y >"$incomplete/claude.log" 2>&1) \
365
- && ! (cd "$incomplete/codex" \
366
- && HOME="$incomplete/home" node "$incomplete/package/bin/devlyn.js" agents codex >"$incomplete/codex.log" 2>&1) \
367
- && grep -Fq 'Incomplete devlyn skill install; missing: devlyn:queue' "$incomplete/claude.log" \
368
- && grep -Fq 'Incomplete devlyn skill install; missing: devlyn:queue' "$incomplete/codex.log" \
369
- && [ -d "$incomplete/claude/.claude/skills/devlyn:resolve" ] \
370
- && [ -d "$incomplete/home/.codex/skills/devlyn:resolve" ] \
371
- && [ ! -e "$incomplete/claude/.claude/skills/.devlyn-install.json" ] \
372
- && [ ! -e "$incomplete/home/.codex/skills/.devlyn-install.json" ]; then
373
- ok "incomplete Claude and Codex copies fail visibly without stale or replacement markers"
374
- else
375
- bad "incomplete Claude and Codex copies must fail visibly without an install marker"
376
- fi
377
- rm -rf "$tmp_install_marker"
378
- else
379
- bad "could not allocate install-marker smoke directory"
380
- fi
381
-
382
- # ---------------------------------------------------------------------------
383
- # 6. Source ↔ installed mirror parity on critical path.
384
- # Only runs if .claude/skills exists (i.e. installer has been run).
385
- # ---------------------------------------------------------------------------
386
- section "Check 6: Source ↔ installed mirror parity (critical path)"
387
- check_skill_mirror_parity \
388
- ".claude/skills" \
389
- "no .claude/skills (fresh checkout) — skipping parity check" \
390
- "missing file on critical path" \
391
- "source and installed differ" \
392
- "critical path parity clean"
393
-
394
- # Codex / agent runtimes in this repo can also expose a project-local
395
- # `.agents/skills` mirror. If it exists, keep the same critical path in parity;
396
- # otherwise a session can read stale pair/risk-probe contracts even while the
397
- # source and `.claude/skills` mirrors are clean.
398
- section "Check 6a: Source ↔ .agents mirror parity (critical path)"
399
- check_skill_mirror_parity \
400
- ".agents/skills" \
401
- "no .agents/skills — skipping parity check" \
402
- "missing .agents critical-path file" \
403
- "source and .agents mirror differ" \
404
- ".agents critical path parity clean"
405
-
406
- # ---------------------------------------------------------------------------
407
- # 6b. VERIFY merge verdict binding self-test.
408
- for helper in role-config judge-role-evidence task-complete; do
409
- if python3 "config/skills/_shared/$helper.py" --self-test; then
410
- ok "$helper.py self-test passed"
411
- else
412
- bad "$helper.py self-test failed"
413
- fi
414
- done
415
-
416
- # F23 full-pipeline prompt-fix rerun exposed a real failure where Codex
417
- # pair-JUDGE emitted HIGH findings but state kept pair_judge as
418
- # PASS_WITH_ISSUES. Routing severity must be deterministic, not prose.
419
- # ---------------------------------------------------------------------------
420
- section "Check 6b: VERIFY merge makes pair HIGH verdict-binding"
421
- if python3 config/skills/_shared/verify-merge-findings.py --self-test >/dev/null 2>&1; then
422
- ok "verify-merge-findings.py self-test passed"
423
- else
424
- bad "verify-merge-findings.py self-test failed"
425
- fi
426
-
427
- if ! grep -Fq 'def pair_trigger_skip_contract_violation' config/skills/_shared/verify-merge-findings.py \
428
- || ! grep -Fq 'def pair_trigger_missing_contract_violation' config/skills/_shared/verify-merge-findings.py \
429
- || ! grep -Fq 'def pair_trigger_reason_completeness_violation' config/skills/_shared/verify-merge-findings.py \
430
- || ! grep -Fq 'def pair_trigger_present' config/skills/_shared/verify-merge-findings.py \
431
- || ! grep -Fq 'KNOWN_PAIR_TRIGGER_REASONS = {' config/skills/_shared/verify-merge-findings.py \
432
- || ! grep -Fq 'mode.pair-verify' config/skills/_shared/verify-merge-findings.py \
433
- || ! grep -Fq 'state.get("pair_verify") is True' config/skills/_shared/verify-merge-findings.py \
434
- || ! grep -Fq 'def pair_flag_contract_violation' config/skills/_shared/verify-merge-findings.py \
435
- || ! grep -Fq 'verify-pair-trigger-conflicting-pair-flags' config/skills/_shared/verify-merge-findings.py \
436
- || ! grep -Fq -- '--pair-verify and --no-pair are mutually exclusive' config/skills/_shared/verify-merge-findings.py \
437
- || ! grep -Fq 'def has_known_pair_trigger_reason' config/skills/_shared/verify-merge-findings.py \
438
- || ! grep -Fq 'def all_known_pair_trigger_reasons' config/skills/_shared/verify-merge-findings.py \
439
- || ! grep -Fq 'return reason in KNOWN_PAIR_TRIGGER_REASONS' config/skills/_shared/verify-merge-findings.py \
440
- || ! grep -Fq '"reasons": ["risk.high", 3]' config/skills/_shared/verify-merge-findings.py \
441
- || ! grep -Fq '"reasons": ["risk.high", "looks-hard"]' config/skills/_shared/verify-merge-findings.py \
442
- || ! grep -Fq '"reasons": ["risk high"]' config/skills/_shared/verify-merge-findings.py \
443
- || ! grep -Fq '"reasons": ["risk_profile.high_risk", "risk_probes_enabled"]' config/skills/_shared/verify-merge-findings.py \
444
- || ! grep -Fq 'verify-pair-trigger-reasons-unknown' config/skills/_shared/verify-merge-findings.py \
445
- || ! grep -Fq 'verify-pair-trigger-reasons-incomplete' config/skills/_shared/verify-merge-findings.py \
446
- || ! grep -Fq 'pair_trigger.reasons is missing applicable canonical reason(s)' config/skills/_shared/verify-merge-findings.py \
447
- || ! grep -Fq 'pair_trigger.reasons must include a known pair-trigger reason' config/skills/_shared/verify-merge-findings.py \
448
- || ! grep -Fq 'pair_trigger.reasons must only include known pair-trigger reasons' config/skills/_shared/verify-merge-findings.py \
449
- || ! grep -Fq 'verify-pair-trigger-ineligible-unjustified' config/skills/_shared/verify-merge-findings.py \
450
- || ! grep -Fq 'VERIFY state requires a pair decision' config/skills/_shared/verify-merge-findings.py \
451
- || ! grep -Fq 'verify-pair-trigger-user-no-pair-unsupported' config/skills/_shared/verify-merge-findings.py \
452
- || ! grep -Fq 'risk_profile.pair_default_enabled false from an explicit --no-pair opt-out' config/skills/_shared/verify-merge-findings.py \
453
- || ! grep -Fq 'def risk_profile_contract_violation' config/skills/_shared/verify-merge-findings.py \
454
- || ! grep -Fq 'verify-risk-profile-malformed' config/skills/_shared/verify-merge-findings.py \
455
- || ! grep -Fq 'risk_profile.risk_probes_enabled must be a boolean' config/skills/_shared/verify-merge-findings.py \
456
- || ! grep -Fq 'risk_profile.reasons must be a list of strings' config/skills/_shared/verify-merge-findings.py \
457
- || ! grep -Fq 'def spec_frontmatter_complexity' config/skills/_shared/verify-merge-findings.py \
458
- || ! grep -Fq '"complexity": "large"' config/skills/_shared/verify-merge-findings.py \
459
- || ! grep -Fq 'complexity.large' config/skills/_shared/verify-merge-findings.py \
460
- || ! grep -Fq 'spec.complexity.high' config/skills/_shared/verify-merge-findings.py \
461
- || ! grep -Fq 'spec.complexity.large' config/skills/_shared/verify-merge-findings.py \
462
- || ! grep -Fq 'verify-pair-trigger-required-missing' config/skills/_shared/verify-merge-findings.py \
463
- || ! grep -Fq 'verify-pair-trigger-skipped-reason-unsupported' config/skills/_shared/verify-merge-findings.py \
464
- || ! grep -Fq 'verify-pair-trigger-mechanical-blocker-unsupported' config/skills/_shared/verify-merge-findings.py \
465
- || ! grep -Fq 'verify-pair-trigger-primary-judge-blocker-unsupported' config/skills/_shared/verify-merge-findings.py \
466
- || ! grep -Fq 'Missing, contradictory, incomplete, or unknown trigger state BLOCKs VERIFY' config/skills/devlyn:resolve/references/state-schema.md \
467
- || ! grep -Fq 'Canonical reasons are `pair.default`, `mode.verify-only`' config/skills/devlyn:resolve/references/state-schema.md \
468
- || ! grep -Fq '`mode.pair-verify`' config/skills/devlyn:resolve/references/state-schema.md \
469
- || ! grep -Fq 'Eligible schema-v3 state must contain `pair.default` and every other applicable canonical' config/skills/devlyn:resolve/references/phases/verify.md \
470
- || ! grep -Fq 'with `pair.default` plus every applicable outcome-independent canonical reason' config/skills/devlyn:resolve/references/phases/verify.md \
471
- || ! grep -Fq '`mode.pair-verify`' config/skills/devlyn:resolve/references/phases/verify.md \
472
- || ! grep -Fq '"pair_verify": false' config/skills/devlyn:resolve/references/state-schema.md \
473
- || ! grep -Fq 'Schema v3.0 eligible state requires `pair.default` plus every applicable telemetry reason' config/skills/devlyn:resolve/references/state-schema.md \
474
- || ! grep -Fq 'Ineligible new-run state has empty reasons and only' config/skills/devlyn:resolve/references/state-schema.md \
475
- || ! grep -Fq 'True only for `--pair-verify`' config/skills/devlyn:resolve/references/state-schema.md \
476
- || ! grep -Fq 'pair_verify: true` only when `--pair-verify` was passed' config/skills/devlyn:resolve/SKILL.md \
477
- || ! grep -Fq 'persist `pair_trigger` before spawn with `pair.default` plus every applicable outcome-independent reason' config/skills/devlyn:resolve/SKILL.md \
478
- || ! grep -Fq '`--pair-verify` and `--no-pair` are mutually exclusive' config/skills/devlyn:resolve/SKILL.md \
479
- || ! grep -Fq 'mutually exclusive with `risk_profile.pair_default_enabled == false`' config/skills/devlyn:resolve/references/state-schema.md \
480
- || ! grep -Fq 'if both are present, stop with `BLOCKED:invalid-flags`' config/skills/devlyn:resolve/references/phases/verify.md \
481
- || ! grep -Fq 'Malformed risk or trigger state BLOCKs VERIFY' config/skills/devlyn:resolve/SKILL.md \
482
- || ! grep -Fq 'pair_default_enabled` is false only for explicit `--no-pair`' config/skills/devlyn:resolve/references/state-schema.md \
483
- || ! grep -Fq 'def reject_json_constant' config/skills/_shared/verify-merge-findings.py \
484
- || ! grep -Fq 'loads_strict_json(raw)' config/skills/_shared/verify-merge-findings.py \
485
- || ! grep -Fq 'invalid JSON numeric constant: NaN' config/skills/_shared/verify-merge-findings.py; then
486
- bad "verify-merge-findings.py must block missing pair_trigger and unsupported skip reasons"
487
- fi
488
- verify_merge_risk_profile_guard_missing=0
489
- for file in \
490
- config/skills/_shared/verify-merge-findings.py \
491
- .agents/skills/_shared/verify-merge-findings.py; do
492
- if ! grep -Fq 'def risk_profile_contract_violation' "$file" \
493
- || ! grep -Fq 'verify-risk-profile-malformed' "$file" \
494
- || ! grep -Fq 'risk_profile.risk_probes_enabled must be a boolean' "$file" \
495
- || ! grep -Fq 'risk_profile.reasons must be a list of strings' "$file"; then
496
- bad "$file — verify-merge-findings.py must fail closed on malformed risk_profile"
497
- verify_merge_risk_profile_guard_missing=1
498
- fi
499
- done
500
- if [ $verify_merge_risk_profile_guard_missing -eq 0 ]; then
501
- ok "verify-merge-findings.py risk_profile shape guard is mirrored"
502
- fi
503
-
504
- section "Check 6b1: Process evidence runner is sealed"
505
- if python3 config/skills/_shared/process-evidence.py --self-test >/dev/null 2>&1; then
506
- ok "process-evidence.py self-test passed"
507
- else
508
- bad "process-evidence.py self-test failed"
509
- fi
510
- if ! grep -Fq 'PHASES = {"implement", "build_gate", "verify"}' config/skills/_shared/process-evidence.py \
511
- || ! grep -Fq 'classification' config/skills/_shared/process-evidence.py \
512
- || ! grep -Fq 'validate_bound_carrier' config/skills/_shared/process-evidence.py \
513
- || ! grep -Fq 'validate_summary_commands' config/skills/_shared/process-evidence.py; then
514
- bad "process-evidence.py must preserve phase identity, capability classification, and bound-byte validation"
515
- fi
516
-
517
- section "Check 6b2: Codex invocation receipts are phase-owned"
518
- if python3 config/skills/_shared/invocation-receipt.py --self-test; then
519
- ok "invocation-receipt.py self-test passed"
520
- else
521
- bad "invocation-receipt.py self-test failed"
522
- fi
523
- if ! grep -Fq 'sandbox must remain workspace-write' config/skills/_shared/invocation-receipt.py \
524
- || ! grep -Fq 'SCHEMA_VERSION = "2.0"' config/skills/_shared/invocation-receipt.py \
525
- || ! grep -Fq 'sandbox_workspace_write.network_access' config/skills/_shared/invocation-receipt.py \
526
- || ! grep -Fq 'forbidden Codex bypass flag' config/skills/_shared/invocation-receipt.py \
527
- || ! grep -Fq 'forbidden Codex sandbox: danger-full-access' config/skills/_shared/codex-monitored.sh \
528
- || ! grep -Fq 'DEVLYN_INVOCATION_RECEIPT' config/skills/_shared/codex-monitored.sh; then
529
- bad "Codex invocation receipts must bind phase identity and reject authority widening"
530
- fi
531
-
532
- section "Check 6c: Codex stdout collection writes canonical pair findings"
533
- if python3 config/skills/_shared/collect-codex-findings.py --self-test >/dev/null 2>&1; then
534
- ok "collect-codex-findings.py self-test passed"
535
- else
536
- bad "collect-codex-findings.py self-test failed"
537
- fi
538
- # iter-0082: the frozen collector contract — every negative through every ingress
539
- # path, plus real judge captures replayed against hand-verified expectations.
540
- # The self-test above is unit-shaped; this is the integration bar against real bytes.
541
- if python3 benchmark/ceiling/probes/r-weld-0082/test-collector-contract.py >/dev/null 2>&1; then
542
- ok "collector contract (iter-0082 negatives + real-capture corpus) passed"
543
- else
544
- bad "collector contract failed — run benchmark/ceiling/probes/r-weld-0082/test-collector-contract.py"
545
- fi
546
- if ! grep -Fq 'def reject_json_constant' config/skills/_shared/collect-codex-findings.py \
547
- || ! grep -Fq 'PARSER["collect_stdout"]' config/skills/_shared/collect-codex-findings.py \
548
- || ! grep -Fq 'parse_constant=reject_json_constant' config/skills/_shared/judge-output-parser.py \
549
- || ! grep -Fq 'NaN pair-JUDGE stdout finding' config/skills/_shared/collect-codex-findings.py; then
550
- bad "collect-codex-findings.py must reject non-standard JSON constants in pair-JUDGE stdout"
551
- fi
552
-
553
- section "Check 6c1: Archive preserves pair/risk-probe artifacts safely"
554
- if python3 config/skills/_shared/archive_run.py --self-test >/dev/null 2>&1; then
555
- ok "archive_run.py self-test passed"
556
- else
557
- bad "archive_run.py self-test failed"
558
- fi
559
- if ! grep -Fq 'SAFE_RUN_ID_RE' config/skills/_shared/archive_run.py \
560
- || ! grep -Fq 'run_id must match [A-Za-z0-9_.-]+' config/skills/_shared/archive_run.py \
561
- || ! grep -Fq 'Archive devlyn:resolve run artifacts' config/skills/_shared/archive_run.py \
562
- || grep -Fq 'Archive auto-resolve run artifacts' config/skills/_shared/archive_run.py \
563
- || ! grep -Fq 'invalid JSON numeric constant: NaN' config/skills/_shared/archive_run.py \
564
- || ! grep -Fq '"verify.pair.findings.jsonl"' config/skills/_shared/archive_run.py \
565
- || ! grep -Fq '"verify-merge.summary.json"' config/skills/_shared/archive_run.py \
566
- || ! grep -Fq '"verify.primary.timeout.json"' config/skills/_shared/archive_run.py \
567
- || ! grep -Fq '"surface-close.output.json"' config/skills/_shared/archive_run.py \
568
- || ! grep -Fq '"*-judge.*"' config/skills/_shared/archive_run.py; then
569
- bad "archive_run.py must safely archive pair/risk-probe evidence and reject unsafe run ids"
570
- fi
571
-
572
- section "Check 6c1: Owner phase lifecycle and scope boundaries"
573
- if python3 scripts/test-owner-phases.py; then
574
- ok "owner phase CLI and Git regression tests passed"
575
- else
576
- bad "owner phase CLI and Git regression tests failed"
577
- fi
578
-
579
- section "Check 6c2: SURFACE_CLOSE v6 dispatch is complete and mirrored"
580
- surface_close_missing=0
581
- if python3 config/skills/_shared/state-phase-write.py --self-test >/dev/null 2>&1; then
582
- ok "SURFACE_CLOSE state/envelope/adjudication/audit/rollback self-test passed"
583
- else
584
- bad "state-phase-write.py SURFACE_CLOSE self-test failed"
585
- fi
586
- if python3 - <<'PY'
587
- import ast
588
- import runpy
589
-
590
- entries = ["src/{a,b}/**"]
591
- paths = ("src/a/example.py", "src/b/example.py", "src/c/example.py")
592
- expected = (True, True, False)
593
- sources = {"path_matches_surface": [], "validate_surface_brace_glob": []}
594
- for name in ("state-phase-write.py", "spec-verify-check.py"):
595
- path = f"config/skills/_shared/{name}"
596
- module = runpy.run_path(path)
597
- actual = tuple(module["path_matches_surface"](path, entries) for path in paths)
598
- assert actual == expected, (name, actual)
599
- tree = ast.parse(open(path, encoding="utf-8").read())
600
- text = open(path, encoding="utf-8").read()
601
- for function_name in sources:
602
- function = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == function_name)
603
- sources[function_name].append(ast.get_source_segment(text, function))
604
- for invalid in ("src/{a,b", "src/{a,}/**", "src/{a,{b,c}}", "src/{a,**}"):
605
- try:
606
- module["path_matches_surface"]("src/a/example.py", [invalid])
607
- except ValueError as exc:
608
- assert invalid in str(exc) and "supported form" in str(exc)
609
- else:
610
- raise AssertionError((name, invalid))
611
- for function_name, bodies in sources.items():
612
- assert bodies[0] == bodies[1], f"{function_name} bodies differ"
613
- PY
614
- then
615
- ok "brace-glob surface matching and errors are byte-parity checked in both consumers"
616
- else
617
- bad "brace-glob surface matching or errors drifted between state and spec checks"
618
- fi
619
- if ! grep -Fq 'rollback_surface_delta' config/skills/_shared/state-phase-write.py \
620
- || ! grep -Fq 'validate_surface_adjudication' config/skills/_shared/state-phase-write.py \
621
- || ! grep -Fq 'validate_surface_execution' config/skills/_shared/state-phase-write.py \
622
- || ! grep -Fq 'phases.surface_close spawn requires --engine claude' config/skills/_shared/state-phase-write.py \
623
- || ! grep -Fq 'SURFACE_CLOSE requires canonical native JSON' config/skills/_shared/state-phase-write.py \
624
- || ! grep -Fq 'BLOCKED:surface-close-input-mismatch' config/skills/_shared/state-phase-write.py \
625
- || ! grep -Fq 'surface-check' config/skills/_shared/state-phase-write.py \
626
- || ! grep -Fq 'surface-rollback' config/skills/_shared/state-phase-write.py; then
627
- bad "state-phase-write.py must mechanically adjudicate, audit, guard, and roll back SURFACE_CLOSE"
628
- fi
629
- for tree in config/skills .agents/skills; do
630
- skill="$tree/devlyn:resolve/SKILL.md"
631
- phase="$tree/devlyn:resolve/references/phases/surface-close.md"
632
- if ! grep -Fq '## PHASE 2.5: SURFACE_CLOSE' "$skill" \
633
- || ! grep -Fq '`state.source.type == "generated"` and complexity is trivial/medium' "$skill" \
634
- || ! grep -Fq 'Engine is Claude always' "$skill" \
635
- || ! grep -Fq 'executor flag/pin' "$skill" \
636
- || ! grep -Fq 'auto_surface_close_claude_unavailable' "$skill" \
637
- || ! grep -Fq 'canonical body VERBATIM' "$skill" \
638
- || ! grep -Fq 'run-bounded.py 600 --stdin-file .devlyn/surface-close.prompt.<round> --record-transport -- claude -p' "$skill" \
639
- || ! grep -Fq -- '--tools "Read,Grep,Glob,Edit,Write" --dangerously-skip-permissions --output-format json --strict-mcp-config --mcp-config '\''{"mcpServers":{}}'\''' "$skill" \
640
- || ! grep -Fq '.devlyn/surface-close.output.json' "$skill" \
641
- || ! grep -Fq '**Common post-fix checkpoint (BUILD_GATE and VERIFY):**' "$skill" \
642
- || ! grep -Fq 'durability-enforce --round <n> --origin-phase <build_gate|cleanup|verify>' "$skill" \
643
- || ! grep -Fq '`surface-rollback`' "$skill" \
644
- || grep -Fq 'Supplied digests' "$phase" \
645
- || grep -Fq 'Hash both artifacts first' "$phase" \
646
- || ! grep -Fq 'Never modify inputs or read state, PLAN, or IMPLEMENT transcript/reasoning.' "$phase" \
647
- || ! grep -Fq 'optionally followed by ` — <one-line evidence>`' "$phase" \
648
- || ! grep -Fq 'N/A <authorized-file>[:<line>] — <one-line evidence-based relationship judgment>' "$phase"; then
649
- bad "$tree — SURFACE_CLOSE v6 dispatch/timeout/adjudication/rollback contract missing"
650
- surface_close_missing=1
651
- fi
652
- done
653
- if [ $surface_close_missing -eq 0 ]; then
654
- ok "SURFACE_CLOSE v6 dispatch/timeout/adjudication/rollback contract is mirrored"
655
- fi
656
-
657
- section "Check 6d: Spec verification executes hidden-blind risk probes"
658
- if python3 config/skills/_shared/spec-verify-check.py --self-test >/dev/null 2>&1; then
659
- ok "spec-verify-check.py risk-probe and expected-contract self-test passed"
660
- else
661
- bad "spec-verify-check.py risk-probe / expected-contract self-test failed"
662
- fi
663
- if ! grep -Fq 'def validate_present_spec_complexity' config/skills/_shared/spec-verify-check.py \
664
- || ! grep -Fq 'SPEC_COMPLEXITY_VALUES = {"trivial", "medium", "high", "large"}' config/skills/_shared/spec-verify-check.py \
665
- || ! grep -Fq 'frontmatter complexity must be one of' config/skills/_shared/spec-verify-check.py \
666
- || ! grep -Fq 'def validate_sibling_spec_complexity' config/skills/_shared/spec-verify-check.py \
667
- || ! grep -Fq 'unsupported spec complexity was accepted' config/skills/_shared/spec-verify-check.py \
668
- || ! grep -Fq 'unsupported sibling spec complexity was accepted by --check-expected' config/skills/_shared/spec-verify-check.py; then
669
- bad "spec-verify-check.py checks must reject unsupported spec complexity values"
670
- else
671
- ok "spec-verify-check.py checks reject unsupported spec complexity values"
672
- fi
673
- if ! grep -Fq 'generated criteria carrier was not staged into .devlyn/spec-verify.json' config/skills/_shared/spec-verify-check.py \
674
- || ! grep -Fq 'spec source with mismatched source.spec_sha256 was accepted' config/skills/_shared/spec-verify-check.py \
675
- || ! grep -Fq 'spec source with matching source.spec_sha256 was not staged' config/skills/_shared/spec-verify-check.py \
676
- || ! grep -Fq 'source.spec_sha256 mismatch' config/skills/_shared/spec-verify-check.py \
677
- || ! grep -Fq 'generated criteria without a JSON carrier was accepted' config/skills/_shared/spec-verify-check.py \
678
- || ! grep -Fq 'generated criteria without source.criteria_sha256 was accepted' config/skills/_shared/spec-verify-check.py \
679
- || ! grep -Fq 'generated criteria with mismatched source.criteria_sha256 was accepted' config/skills/_shared/spec-verify-check.py \
680
- || ! grep -Fq 'def source_integrity_error' config/skills/_shared/spec-verify-check.py \
681
- || ! grep -Fq 'source.criteria_sha256 mismatch' config/skills/_shared/spec-verify-check.py \
682
- || ! grep -Fq 'Generated criteria were written without one' config/skills/_shared/spec-verify-check.py \
683
- || ! grep -Fq 'generated criteria carrier was not staged into .devlyn/spec-verify.json' .agents/skills/_shared/spec-verify-check.py \
684
- || ! grep -Fq 'spec source with mismatched source.spec_sha256 was accepted' .agents/skills/_shared/spec-verify-check.py \
685
- || ! grep -Fq 'spec source with matching source.spec_sha256 was not staged' .agents/skills/_shared/spec-verify-check.py \
686
- || ! grep -Fq 'source.spec_sha256 mismatch' .agents/skills/_shared/spec-verify-check.py \
687
- || ! grep -Fq 'generated criteria without a JSON carrier was accepted' .agents/skills/_shared/spec-verify-check.py \
688
- || ! grep -Fq 'generated criteria without source.criteria_sha256 was accepted' .agents/skills/_shared/spec-verify-check.py \
689
- || ! grep -Fq 'generated criteria with mismatched source.criteria_sha256 was accepted' .agents/skills/_shared/spec-verify-check.py \
690
- || ! grep -Fq 'def source_integrity_error' .agents/skills/_shared/spec-verify-check.py \
691
- || ! grep -Fq 'source.criteria_sha256 mismatch' .agents/skills/_shared/spec-verify-check.py \
692
- || ! grep -Fq 'Generated criteria were written without one' .agents/skills/_shared/spec-verify-check.py \
693
- || ! grep -Fq '"criteria_sha256": generated_hash' .agents/skills/_shared/spec-verify-check.py; then
694
- bad "spec-verify-check.py self-test must cover generated criteria source extraction"
695
- else
696
- ok "spec-verify-check.py covers generated criteria source extraction"
697
- fi
698
- if ! grep -Fq 'def validate_present_solo_headroom_hypothesis' config/skills/_shared/spec-verify-check.py \
699
- || ! grep -Fq 'def state_requires_risk_probes' config/skills/_shared/spec-verify-check.py \
700
- || ! grep -Fq 'def risk_probes_state_error' config/skills/_shared/spec-verify-check.py \
701
- || ! grep -Fq -- '--include-risk-probes accepted missing required risk-probes.jsonl' config/skills/_shared/spec-verify-check.py \
702
- || ! grep -Fq -- '--include-risk-probes accepted non-boolean risk_probes_enabled' config/skills/_shared/spec-verify-check.py \
703
- || ! grep -Fq -- '--include-risk-probes accepted non-object risk_profile' config/skills/_shared/spec-verify-check.py \
704
- || ! grep -Fq 'def validate_risk_probes_cover_solo_headroom_hypothesis' config/skills/_shared/spec-verify-check.py \
705
- || ! grep -Fq 'def has_backticked_observable_miss_command' config/skills/_shared/spec-verify-check.py \
706
- || ! grep -Fq 'backticked command/observable line that exposes the miss' config/skills/_shared/spec-verify-check.py \
707
- || ! grep -Fq 'weak solo-headroom hypothesis was accepted by --check' config/skills/_shared/spec-verify-check.py \
708
- || ! grep -Fq 'descriptive backtick solo-headroom hypothesis was accepted by --check' config/skills/_shared/spec-verify-check.py \
709
- || ! grep -Fq 'risk probe missing solo-headroom command coverage was accepted' config/skills/_shared/spec-verify-check.py \
710
- || ! grep -Fq 'risk probe with unrelated solo-headroom derived_from was accepted' config/skills/_shared/spec-verify-check.py \
711
- || ! grep -Fq 'risk-probes[0].derived_from must reference the solo-headroom hypothesis bullet' config/skills/_shared/spec-verify-check.py \
712
- || ! grep -Fq 'solo-headroom command in a later risk probe was accepted' config/skills/_shared/spec-verify-check.py \
713
- || ! grep -Fq 'solo-headroom command prefix match was accepted' config/skills/_shared/spec-verify-check.py \
714
- || ! grep -Fq '(?<![A-Za-z0-9_.:/=-])' config/skills/_shared/spec-verify-check.py \
715
- || ! grep -Fq 'risk-probes[0].cmd must contain a solo-headroom hypothesis observable command' config/skills/_shared/spec-verify-check.py \
716
- || ! grep -Fq 'weak sibling solo-headroom hypothesis was accepted by --check-expected' config/skills/_shared/spec-verify-check.py \
717
- || ! grep -Fq 'docs-style solo-headroom hypothesis was rejected by --check' config/skills/_shared/spec-verify-check.py \
718
- || ! grep -Fq 'docs-style sibling solo-headroom command was rejected by --check-expected' config/skills/_shared/spec-verify-check.py \
719
- || ! grep -Fq 'def validate_present_solo_headroom_hypothesis' .agents/skills/_shared/spec-verify-check.py \
720
- || ! grep -Fq 'def state_requires_risk_probes' .agents/skills/_shared/spec-verify-check.py \
721
- || ! grep -Fq 'def risk_probes_state_error' .agents/skills/_shared/spec-verify-check.py \
722
- || ! grep -Fq -- '--include-risk-probes accepted missing required risk-probes.jsonl' .agents/skills/_shared/spec-verify-check.py \
723
- || ! grep -Fq -- '--include-risk-probes accepted non-boolean risk_probes_enabled' .agents/skills/_shared/spec-verify-check.py \
724
- || ! grep -Fq -- '--include-risk-probes accepted non-object risk_profile' .agents/skills/_shared/spec-verify-check.py \
725
- || ! grep -Fq 'def validate_risk_probes_cover_solo_headroom_hypothesis' .agents/skills/_shared/spec-verify-check.py \
726
- || ! grep -Fq 'def has_backticked_observable_miss_command' .agents/skills/_shared/spec-verify-check.py \
727
- || ! grep -Fq 'backticked command/observable line that exposes the miss' .agents/skills/_shared/spec-verify-check.py \
728
- || ! grep -Fq 'weak solo-headroom hypothesis was accepted by --check' .agents/skills/_shared/spec-verify-check.py \
729
- || ! grep -Fq 'descriptive backtick solo-headroom hypothesis was accepted by --check' .agents/skills/_shared/spec-verify-check.py \
730
- || ! grep -Fq 'risk probe missing solo-headroom command coverage was accepted' .agents/skills/_shared/spec-verify-check.py \
731
- || ! grep -Fq 'risk probe with unrelated solo-headroom derived_from was accepted' .agents/skills/_shared/spec-verify-check.py \
732
- || ! grep -Fq 'risk-probes[0].derived_from must reference the solo-headroom hypothesis bullet' .agents/skills/_shared/spec-verify-check.py \
733
- || ! grep -Fq 'solo-headroom command in a later risk probe was accepted' .agents/skills/_shared/spec-verify-check.py \
734
- || ! grep -Fq 'solo-headroom command prefix match was accepted' .agents/skills/_shared/spec-verify-check.py \
735
- || ! grep -Fq '(?<![A-Za-z0-9_.:/=-])' .agents/skills/_shared/spec-verify-check.py \
736
- || ! grep -Fq 'risk-probes[0].cmd must contain a solo-headroom hypothesis observable command' .agents/skills/_shared/spec-verify-check.py \
737
- || ! grep -Fq 'weak sibling solo-headroom hypothesis was accepted by --check-expected' .agents/skills/_shared/spec-verify-check.py \
738
- || ! grep -Fq 'docs-style solo-headroom hypothesis was rejected by --check' .agents/skills/_shared/spec-verify-check.py \
739
- || ! grep -Fq 'docs-style sibling solo-headroom command was rejected by --check-expected' .agents/skills/_shared/spec-verify-check.py; then
740
- bad "spec-verify-check.py --check and --check-expected must reject weak solo-headroom hypotheses"
741
- else
742
- ok "spec-verify-check.py rejects weak solo-headroom hypotheses"
743
- fi
744
- if ! grep -Fq 'requires `.devlyn/risk-probes.jsonl`' config/skills/devlyn:resolve/SKILL.md \
745
- || ! grep -Fq 'requires `.devlyn/risk-probes.jsonl`' .agents/skills/devlyn:resolve/SKILL.md \
746
- || ! grep -Fq 'missing `.devlyn/risk-probes.jsonl` is a CRITICAL mechanical blocker' config/skills/devlyn:resolve/SKILL.md \
747
- || ! grep -Fq 'missing `.devlyn/risk-probes.jsonl` is a CRITICAL mechanical blocker' .agents/skills/devlyn:resolve/SKILL.md \
748
- || ! grep -Fq 'requires that file when `state.risk_profile.risk_probes_enabled == true`' config/skills/devlyn:resolve/references/phases/build-gate.md \
749
- || ! grep -Fq 'requires that file when `state.risk_profile.risk_probes_enabled == true`' .agents/skills/devlyn:resolve/references/phases/build-gate.md \
750
- || ! grep -Fq 'Malformed `state.risk_profile` is also CRITICAL because it can hide enabled risk probes' config/skills/devlyn:resolve/references/phases/build-gate.md \
751
- || ! grep -Fq 'Malformed `state.risk_profile` is also CRITICAL because it can hide enabled risk probes' .agents/skills/devlyn:resolve/references/phases/build-gate.md \
752
- || ! grep -Fq 'When `state.risk_profile.risk_probes_enabled == true`, missing `.devlyn/risk-probes.jsonl` is also CRITICAL' config/skills/devlyn:resolve/references/phases/verify.md \
753
- || ! grep -Fq 'When `state.risk_profile.risk_probes_enabled == true`, missing `.devlyn/risk-probes.jsonl` is also CRITICAL' .agents/skills/devlyn:resolve/references/phases/verify.md; then
754
- bad "BUILD_GATE and VERIFY must fail closed when enabled risk probes are missing"
755
- else
756
- ok "BUILD_GATE and VERIFY require enabled risk probes"
757
- fi
758
- if grep -Fq 'or any(char.isspace() for char in stripped)' config/skills/_shared/spec-verify-check.py \
759
- || grep -Fq 'or any(char.isspace() for char in stripped)' .agents/skills/_shared/spec-verify-check.py \
760
- || grep -Fq 'or any(char.isspace() for char in stripped)' config/skills/_shared/verify-merge-findings.py \
761
- || grep -Fq 'or any(char.isspace() for char in stripped)' .agents/skills/_shared/verify-merge-findings.py \
762
- || grep -Fq 'or any(char.isspace() for char in stripped)' benchmark/auto-resolve/scripts/pair_evidence_contract.py; then
763
- bad "solo-headroom command detection must not treat descriptive whitespace as a command"
764
- else
765
- ok "solo-headroom command detection rejects descriptive whitespace"
766
- fi
767
- if ! grep -Fq '"printf",' config/skills/_shared/spec-verify-check.py \
768
- || ! grep -Fq '"printf",' .agents/skills/_shared/spec-verify-check.py \
769
- || ! grep -Fq '"printf",' config/skills/_shared/verify-merge-findings.py \
770
- || ! grep -Fq '"printf",' .agents/skills/_shared/verify-merge-findings.py \
771
- || ! grep -Fq '"printf",' benchmark/auto-resolve/scripts/pair_evidence_contract.py; then
772
- bad "solo-headroom command detection must keep explicit printf command support"
773
- else
774
- ok "solo-headroom command detection keeps explicit printf command support"
775
- fi
776
- if python3 - <<'PY'
777
- import ast
778
- import pathlib
779
- import sys
780
-
781
- files = [
782
- pathlib.Path("benchmark/auto-resolve/scripts/pair_evidence_contract.py"),
783
- pathlib.Path("config/skills/_shared/spec-verify-check.py"),
784
- pathlib.Path(".agents/skills/_shared/spec-verify-check.py"),
785
- pathlib.Path("config/skills/_shared/verify-merge-findings.py"),
786
- pathlib.Path(".agents/skills/_shared/verify-merge-findings.py"),
787
- ]
788
- names = [
789
- "COMMAND_PREFIXES",
790
- "RESERVED_BACKTICK_TERMS",
791
- "OBSERVABLE_COMMAND_MARKERS",
792
- ]
793
-
794
- def extract(path: pathlib.Path, name: str) -> tuple[str, ...]:
795
- tree = ast.parse(path.read_text(encoding="utf-8"))
796
- for node in tree.body:
797
- if isinstance(node, ast.Assign):
798
- if any(isinstance(target, ast.Name) and target.id == name for target in node.targets):
799
- value = ast.literal_eval(node.value)
800
- return tuple(sorted(value))
801
- raise AssertionError(f"{path}: missing {name}")
802
-
803
- baseline = {name: extract(files[0], name) for name in names}
804
- for path in files[1:]:
805
- for name in names:
806
- value = extract(path, name)
807
- if value != baseline[name]:
808
- print(f"{path}: {name} drifted from {files[0]}", file=sys.stderr)
809
- print(f"expected={baseline[name]!r}", file=sys.stderr)
810
- print(f"actual={value!r}", file=sys.stderr)
811
- sys.exit(1)
812
- PY
813
- then
814
- ok "solo-headroom command detection constants stay in parity"
815
- else
816
- bad "solo-headroom command detection constants must stay in parity"
817
- fi
818
- if ! grep -Fq 'def resolve_required_risk_probe_requirements' config/skills/_shared/spec-verify-check.py \
819
- || ! grep -Fq 'def validate_required_risk_probe_requirement' config/skills/_shared/spec-verify-check.py \
820
- || ! grep -Fq 'missing required probe(s)' config/skills/_shared/spec-verify-check.py \
821
- || ! grep -Fq 'risk-probes.jsonl missing a declared required_risk_probe_requirements' config/skills/_shared/spec-verify-check.py \
822
- || ! grep -Fq 'risk-probes.jsonl covering a declared required_risk_probe_requirements' config/skills/_shared/spec-verify-check.py \
823
- || ! grep -Fq 'required_risk_probe_requirements with an unknown tag was accepted' config/skills/_shared/spec-verify-check.py \
824
- || ! grep -Fq 'required_risk_probe_requirements.derived_from not present in the' config/skills/_shared/spec-verify-check.py \
825
- || ! grep -Fq 'required_risk_probe_requirements' config/skills/_shared/expected.schema.json \
826
- || ! grep -Fq 'required_risk_probe_requirements' config/skills/devlyn:ideate/references/spec-template.md; then
827
- bad "spec-verify-check.py must enforce declared required_risk_probe_requirements (iter-0049 language-neutral F3 replacement)"
828
- fi
829
- if ! grep -Fq 'spec.expected.json top-level array produced a traceback' config/skills/_shared/spec-verify-check.py \
830
- || ! grep -Fq 'invalid spec.expected.json produced a traceback' config/skills/_shared/spec-verify-check.py \
831
- || ! grep -Fq 'NaN spec.expected.json did not report invalid numeric constant' config/skills/_shared/spec-verify-check.py \
832
- || ! grep -Fq 'NaN risk-probes JSONL did not report invalid numeric constant' config/skills/_shared/spec-verify-check.py \
833
- || ! grep -Fq 'def reject_json_constant' config/skills/_shared/spec-verify-check.py \
834
- || ! grep -Fq 'loads_strict_json(line)' config/skills/_shared/spec-verify-check.py \
835
- || ! grep -Fq 'loads_strict_json(expected_path.read_text(encoding="utf-8"))' config/skills/_shared/spec-verify-check.py \
836
- || ! grep -Fq 'top-level must be a JSON object' config/skills/_shared/spec-verify-check.py \
837
- || ! grep -Fq 'has invalid JSON' config/skills/_shared/spec-verify-check.py; then
838
- bad "spec-verify-check.py self-test must fail malformed spec.expected.json cleanly without traceback"
839
- fi
840
- if ! grep -Fq '"asserts_named_stream_output"' config/skills/_shared/spec-verify-check.py \
841
- || ! grep -Fq '"asserts_error_payload_or_stderr"' config/skills/_shared/spec-verify-check.py \
842
- || ! grep -Fq '"asserts_nonzero_or_exit_2"' config/skills/_shared/spec-verify-check.py \
843
- || ! grep -Fq 'error_contract without exit-code evidence was accepted' config/skills/_shared/spec-verify-check.py \
844
- || ! grep -Fq 'stdout_stderr_contract without stream evidence was accepted' config/skills/_shared/spec-verify-check.py \
845
- || ! grep -Fq '`stdout_stderr_contract`: `asserts_named_stream_output`' config/skills/devlyn:resolve/references/phases/probe-derive.md \
846
- || ! grep -Fq '`error_contract`: `asserts_error_payload_or_stderr`' config/skills/devlyn:resolve/references/phases/probe-derive.md \
847
- || ! grep -Fq '`asserts_nonzero_or_exit_2`' config/skills/devlyn:resolve/SKILL.md; then
848
- bad "risk-probe error/stdout-stderr tags must require concrete tag_evidence markers in validator and prompt contract"
849
- fi
850
- if ! grep -Fq '"http_error_contract"' config/skills/_shared/spec-verify-check.py \
851
- || ! grep -Fq 'asserts_http_error_status' config/skills/_shared/spec-verify-check.py \
852
- || ! grep -Fq 'http_error_contract without payload evidence was accepted' config/skills/_shared/spec-verify-check.py \
853
- || ! grep -Fq 'exact error body shape_contract without exact object evidence was accepted' config/skills/_shared/spec-verify-check.py \
854
- || ! grep -Fq 'exact error body shape_contract with exact object evidence was rejected' config/skills/_shared/spec-verify-check.py \
855
- || ! grep -Fq '`http_error_contract`: `asserts_http_error_status`' config/skills/devlyn:resolve/references/phases/probe-derive.md \
856
- || ! grep -Fq '`http_error_contract` must include `asserts_http_error_status`' config/skills/devlyn:resolve/SKILL.md; then
857
- bad "risk-probe HTTP error contracts must require concrete status and payload markers"
858
- fi
859
- if ! grep -Fq '"uses_visible_input_key_names"' config/skills/_shared/spec-verify-check.py \
860
- || ! grep -Fq '"asserts_visible_output_key_names"' config/skills/_shared/spec-verify-check.py \
861
- || ! grep -Fq '"asserts_no_unexpected_output_keys"' config/skills/_shared/spec-verify-check.py \
862
- || ! grep -Fq 'shape_contract claiming visible exact error object without' config/skills/_shared/spec-verify-check.py \
863
- || ! grep -Fq 'visible_text_names_exact_json_error_object' config/skills/_shared/spec-verify-check.py \
864
- || ! grep -Fq 'JSON error object shape_contract with exact object evidence was rejected' config/skills/_shared/spec-verify-check.py \
865
- || ! grep -Fq 'shape_contract without any evidence was accepted' config/skills/_shared/spec-verify-check.py \
866
- || ! grep -Fq 'shape_contract with exact key evidence was rejected' config/skills/_shared/spec-verify-check.py \
867
- || ! grep -Fq '`shape_contract` when the visible text names exact keys' config/skills/devlyn:resolve/references/phases/probe-derive.md \
868
- || ! grep -Fq 'visible_text_names_exact_json_error_object' config/skills/devlyn:resolve/references/phases/probe-derive.md \
869
- || ! grep -Fq '`shape_contract` must' config/skills/devlyn:resolve/SKILL.md \
870
- || ! grep -Fq 'visible_text_names_exact_json_error_object' config/skills/devlyn:resolve/SKILL.md; then
871
- bad "risk-probe shape contracts must require exact visible input/output key evidence unconditionally, and asserts_exact_error_object when the probe claims the visible text names an exact JSON error object"
872
- fi
873
- if ! grep -Fq '"auth_signature_contract"' config/skills/_shared/spec-verify-check.py \
874
- || ! grep -Fq '"idempotency_replay"' config/skills/_shared/spec-verify-check.py \
875
- || ! grep -Fq 'asserts_signature_over_exact_bytes' config/skills/_shared/spec-verify-check.py \
876
- || ! grep -Fq 'duplicate_id_rejected_regardless_of_body' config/skills/_shared/spec-verify-check.py \
877
- || ! grep -Fq '`auth_signature_contract`: `asserts_signature_over_exact_bytes`' config/skills/devlyn:resolve/references/phases/probe-derive.md \
878
- || ! grep -Fq '`idempotency_replay`: `first_delivery_then_duplicate`' config/skills/devlyn:resolve/references/phases/probe-derive.md \
879
- || ! grep -Fq '`auth_signature_contract` must include `asserts_signature_over_exact_bytes`' config/skills/devlyn:resolve/SKILL.md; then
880
- bad "risk-probe webhook/signature/replay contracts must require concrete auth_signature_contract and idempotency_replay tags"
881
- fi
882
- if ! grep -Fq '"concurrent_state_consistency"' config/skills/_shared/spec-verify-check.py \
883
- || ! grep -Fq 'overlapping_mutations_exercised' config/skills/_shared/spec-verify-check.py \
884
- || ! grep -Fq '`concurrent_state_consistency`: `overlapping_mutations_exercised`' config/skills/devlyn:resolve/references/phases/probe-derive.md \
885
- || ! grep -Fq '`concurrent_state_consistency` must' config/skills/devlyn:resolve/SKILL.md; then
886
- bad "risk-probe concurrent state contracts must require concrete concurrent_state_consistency markers"
887
- fi
888
- if ! grep -Fq '"atomic_batch_state"' config/skills/_shared/spec-verify-check.py \
889
- || ! grep -Fq 'mixed_valid_invalid_batch' config/skills/_shared/spec-verify-check.py \
890
- || ! grep -Fq 'atomic_batch_state without success-order evidence was accepted' config/skills/_shared/spec-verify-check.py \
891
- || ! grep -Fq '`atomic_batch_state`: `mixed_valid_invalid_batch`' config/skills/devlyn:resolve/references/phases/probe-derive.md \
892
- || ! grep -Fq '`atomic_batch_state` must include `mixed_valid_invalid_batch`' config/skills/devlyn:resolve/SKILL.md; then
893
- bad "risk-probe atomic batch contracts must require concrete mixed-failure and success-order markers"
894
- fi
895
-
896
- section "Check 6f: ideate validates sibling spec.expected.json"
897
- expected_check_missing=0
898
- for file in \
899
- config/skills/devlyn:ideate/SKILL.md \
900
- config/skills/devlyn:ideate/references/elicitation.md \
901
- config/skills/devlyn:ideate/references/from-spec-mode.md \
902
- config/skills/devlyn:ideate/references/project-mode.md \
903
- config/skills/devlyn:ideate/references/spec-template.md
904
- do
905
- if ! grep -Fq -- '--check-expected <expected-path>' "$file"; then
906
- bad "$file — missing spec.expected.json mechanical validation command"
907
- expected_check_missing=1
908
- fi
909
- done
910
- if [ $expected_check_missing -eq 0 ]; then
911
- ok "ideate docs require --check-expected for sibling expected contracts"
912
- fi
913
- if ! grep -Fq 'any present actionable solo-headroom hypothesis' config/skills/devlyn:ideate/SKILL.md \
914
- || ! grep -Fq 'any present actionable solo-headroom hypothesis' .agents/skills/devlyn:ideate/SKILL.md \
915
- || ! grep -Fq 'any present actionable solo-headroom hypothesis' config/skills/devlyn:ideate/references/elicitation.md \
916
- || ! grep -Fq 'any present actionable solo-headroom hypothesis' .agents/skills/devlyn:ideate/references/elicitation.md \
917
- || ! grep -Fq 'any present actionable solo-headroom hypothesis' config/skills/devlyn:ideate/references/project-mode.md \
918
- || ! grep -Fq 'any present actionable solo-headroom hypothesis' .agents/skills/devlyn:ideate/references/project-mode.md \
919
- || ! grep -Fq 'any present actionable solo-headroom hypothesis' config/skills/devlyn:resolve/SKILL.md \
920
- || ! grep -Fq 'any present actionable solo-headroom hypothesis' .agents/skills/devlyn:resolve/SKILL.md \
921
- || ! grep -Fq "legacy inline \`## Verification\` JSON carrier" config/skills/devlyn:ideate/SKILL.md \
922
- || ! grep -Fq "legacy inline \`## Verification\` JSON carrier" .agents/skills/devlyn:ideate/SKILL.md \
923
- || ! grep -Fq "legacy inline \`## Verification\` JSON carrier" config/skills/devlyn:ideate/references/elicitation.md \
924
- || ! grep -Fq "legacy inline \`## Verification\` JSON carrier" .agents/skills/devlyn:ideate/references/elicitation.md \
925
- || ! grep -Fq "legacy inline \`## Verification\` JSON carrier" config/skills/devlyn:ideate/references/from-spec-mode.md \
926
- || ! grep -Fq "legacy inline \`## Verification\` JSON carrier" .agents/skills/devlyn:ideate/references/from-spec-mode.md \
927
- || ! grep -Fq "inline \`## Verification\` JSON carrier" config/skills/devlyn:resolve/SKILL.md \
928
- || ! grep -Fq "inline \`## Verification\` JSON carrier" .agents/skills/devlyn:resolve/SKILL.md \
929
- || ! grep -Fq 'spec.expected.json.verification_commands[].cmd' config/skills/devlyn:ideate/SKILL.md \
930
- || ! grep -Fq 'spec.expected.json.verification_commands[].cmd' .agents/skills/devlyn:ideate/SKILL.md \
931
- || ! grep -Fq 'spec.expected.json.verification_commands[].cmd' config/skills/devlyn:ideate/references/elicitation.md \
932
- || ! grep -Fq 'spec.expected.json.verification_commands[].cmd' .agents/skills/devlyn:ideate/references/elicitation.md \
933
- || ! grep -Fq 'spec.expected.json.verification_commands[].cmd' config/skills/devlyn:ideate/references/project-mode.md \
934
- || ! grep -Fq 'spec.expected.json.verification_commands[].cmd' .agents/skills/devlyn:ideate/references/project-mode.md \
935
- || ! grep -Fq 'spec.expected.json.verification_commands[].cmd' config/skills/devlyn:resolve/SKILL.md \
936
- || ! grep -Fq 'spec.expected.json.verification_commands[].cmd' .agents/skills/devlyn:resolve/SKILL.md; then
937
- bad "ideate/resolve docs must describe mechanical solo-headroom validation"
938
- else
939
- ok "ideate/resolve docs describe mechanical solo-headroom validation"
940
- fi
941
- if ! grep -Fq 'def validate_expected_against_sibling_spec' config/skills/_shared/spec-verify-check.py \
942
- || ! grep -Fq 'empty verification_commands should fail for runtime specs' config/skills/_shared/spec-verify-check.py \
943
- || ! grep -Fq 'empty verification_commands should be valid for pure-design specs' config/skills/_shared/spec-verify-check.py \
944
- || ! grep -Fq 'backticked_observable_miss_commands(spec_text)' config/skills/_shared/spec-verify-check.py \
945
- || ! grep -Fq 'observable command must match spec.expected.json' config/skills/_shared/spec-verify-check.py \
946
- || ! grep -Fq 'observable command must match `## Verification` JSON carrier' config/skills/_shared/spec-verify-check.py \
947
- || ! grep -Fq 'mismatched inline solo-headroom command was accepted by --check' config/skills/_shared/spec-verify-check.py \
948
- || ! grep -Fq 'matched inline solo-headroom command was rejected by --check' config/skills/_shared/spec-verify-check.py \
949
- || ! grep -Fq 'mismatched sibling solo-headroom command was accepted by --check-expected' config/skills/_shared/spec-verify-check.py \
950
- || ! grep -Fq 'matched sibling solo-headroom command was rejected by --check-expected' config/skills/_shared/spec-verify-check.py \
951
- || ! grep -Fq 'def validate_present_solo_ceiling_avoidance' config/skills/_shared/spec-verify-check.py \
952
- || ! grep -Fq 'weak solo ceiling avoidance was accepted by --check' config/skills/_shared/spec-verify-check.py \
953
- || ! grep -Fq 'actionable solo ceiling avoidance was rejected by --check' config/skills/_shared/spec-verify-check.py \
954
- || ! grep -Fq 'weak sibling solo ceiling avoidance was accepted by --check-expected' config/skills/_shared/spec-verify-check.py \
955
- || ! grep -Fq 'actionable sibling solo ceiling avoidance was rejected by --check-expected' config/skills/_shared/spec-verify-check.py \
956
- || ! grep -Fq 'backticked_observable_miss_commands(spec_text)' .agents/skills/_shared/spec-verify-check.py \
957
- || ! grep -Fq 'observable command must match spec.expected.json' .agents/skills/_shared/spec-verify-check.py \
958
- || ! grep -Fq 'observable command must match `## Verification` JSON carrier' .agents/skills/_shared/spec-verify-check.py \
959
- || ! grep -Fq 'mismatched inline solo-headroom command was accepted by --check' .agents/skills/_shared/spec-verify-check.py \
960
- || ! grep -Fq 'matched inline solo-headroom command was rejected by --check' .agents/skills/_shared/spec-verify-check.py \
961
- || ! grep -Fq 'mismatched sibling solo-headroom command was accepted by --check-expected' .agents/skills/_shared/spec-verify-check.py \
962
- || ! grep -Fq 'matched sibling solo-headroom command was rejected by --check-expected' .agents/skills/_shared/spec-verify-check.py \
963
- || ! grep -Fq 'def validate_present_solo_ceiling_avoidance' .agents/skills/_shared/spec-verify-check.py \
964
- || ! grep -Fq 'weak solo ceiling avoidance was accepted by --check' .agents/skills/_shared/spec-verify-check.py \
965
- || ! grep -Fq 'actionable solo ceiling avoidance was rejected by --check' .agents/skills/_shared/spec-verify-check.py \
966
- || ! grep -Fq 'weak sibling solo ceiling avoidance was accepted by --check-expected' .agents/skills/_shared/spec-verify-check.py \
967
- || ! grep -Fq 'actionable sibling solo ceiling avoidance was rejected by --check-expected' .agents/skills/_shared/spec-verify-check.py; then
968
- bad "spec-verify-check.py must reject empty expected runtime contracts, weak solo ceiling avoidance, and preserve pure-design escape"
969
- fi
970
- if ! grep -Fq 'Verification includes at least one compound scenario that exercises the interaction end-to-end' \
971
- config/skills/devlyn:ideate/references/spec-template.md \
972
- || ! grep -Fq 'Verification includes at least one compound scenario that exercises the interaction end-to-end' .agents/skills/devlyn:ideate/references/spec-template.md; then
973
- bad "ideate spec template must require compound interaction verification for pair-relevant high-risk specs"
974
- else
975
- ok "ideate spec template requires compound interaction verification for pair-relevant specs"
976
- fi
977
- if ! grep -Fq 'ask for one concrete compound' config/skills/devlyn:ideate/references/elicitation.md \
978
- || ! grep -Fq 'ask for one concrete compound' .agents/skills/devlyn:ideate/references/elicitation.md; then
979
- bad "ideate elicitation must ask for compound interaction scenarios when pair-relevant risks appear"
980
- else
981
- ok "ideate elicitation asks for compound interaction scenarios when pair-relevant risks appear"
982
- fi
983
- if ! grep -Fq 'solo-headroom hypothesis inside `## Verification`' config/skills/devlyn:ideate/references/spec-template.md \
984
- || ! grep -Fq 'solo-headroom hypothesis inside `## Verification`' .agents/skills/devlyn:ideate/references/spec-template.md \
985
- || ! grep -Fq 'ask for one solo-headroom hypothesis' config/skills/devlyn:ideate/references/elicitation.md \
986
- || ! grep -Fq 'ask for one solo-headroom hypothesis' .agents/skills/devlyn:ideate/references/elicitation.md; then
987
- bad "ideate must require a visible solo-headroom hypothesis for benchmark and pair-evidence specs"
988
- else
989
- ok "ideate requires solo-headroom hypothesis for benchmark and pair-evidence specs"
990
- fi
991
- if ! grep -Fq 'must literally contain `solo-headroom hypothesis`' config/skills/devlyn:ideate/references/spec-template.md \
992
- || ! grep -Fq 'must literally contain `solo-headroom hypothesis`' .agents/skills/devlyn:ideate/references/spec-template.md \
993
- || ! grep -Fq 'backticked line itself must contain `miss`' config/skills/devlyn:ideate/references/spec-template.md \
994
- || ! grep -Fq 'backticked line itself must contain `miss`' .agents/skills/devlyn:ideate/references/spec-template.md \
995
- || ! grep -Fq '`solo_claude`, `miss`, and a backticked observable command' config/skills/devlyn:ideate/references/spec-template.md \
996
- || ! grep -Fq '`solo_claude`, `miss`, and a backticked observable command' .agents/skills/devlyn:ideate/references/spec-template.md \
997
- || ! grep -Fq 'command/observable' config/skills/devlyn:ideate/references/spec-template.md \
998
- || ! grep -Fq 'command/observable' .agents/skills/devlyn:ideate/references/spec-template.md \
999
- || ! grep -Fq 'must literally contain `solo-headroom hypothesis`' config/skills/devlyn:ideate/references/elicitation.md \
1000
- || ! grep -Fq 'must literally contain `solo-headroom hypothesis`' .agents/skills/devlyn:ideate/references/elicitation.md \
1001
- || ! grep -Fq 'line itself must contain `miss`' config/skills/devlyn:ideate/references/elicitation.md \
1002
- || ! grep -Fq 'line itself must contain `miss`' .agents/skills/devlyn:ideate/references/elicitation.md \
1003
- || ! grep -Fq 'Do not write a benchmark/risk-probe/pair-evidence spec until this' config/skills/devlyn:ideate/references/elicitation.md \
1004
- || ! grep -Fq 'Do not write a benchmark/risk-probe/pair-evidence spec until this' .agents/skills/devlyn:ideate/references/elicitation.md \
1005
- || ! grep -Fq 'spec not ready — solo-headroom hypothesis required' config/skills/devlyn:ideate/references/elicitation.md \
1006
- || ! grep -Fq 'spec not ready — solo-headroom hypothesis required' .agents/skills/devlyn:ideate/references/elicitation.md \
1007
- || ! grep -Fq '`solo_claude`, `miss`, and a backticked observable command' config/skills/devlyn:ideate/references/elicitation.md \
1008
- || ! grep -Fq '`solo_claude`, `miss`, and a backticked observable command' .agents/skills/devlyn:ideate/references/elicitation.md \
1009
- || ! grep -Fq 'command/observable' config/skills/devlyn:ideate/references/elicitation.md \
1010
- || ! grep -Fq 'command/observable' .agents/skills/devlyn:ideate/references/elicitation.md \
1011
- || ! grep -Fq 'Verification literally contains `solo-headroom hypothesis`, `solo_claude`' config/skills/devlyn:ideate/references/from-spec-mode.md \
1012
- || ! grep -Fq 'Verification literally contains `solo-headroom hypothesis`, `solo_claude`' .agents/skills/devlyn:ideate/references/from-spec-mode.md \
1013
- || ! grep -Fq 'backticked line itself must contain `miss`' config/skills/devlyn:ideate/references/from-spec-mode.md \
1014
- || ! grep -Fq 'backticked line itself must contain `miss`' .agents/skills/devlyn:ideate/references/from-spec-mode.md \
1015
- || ! grep -Fq 'command/observable' config/skills/devlyn:ideate/references/from-spec-mode.md \
1016
- || ! grep -Fq 'command/observable' .agents/skills/devlyn:ideate/references/from-spec-mode.md \
1017
- || ! grep -Fq 'spec.expected.json.verification_commands[].cmd' config/skills/devlyn:ideate/references/from-spec-mode.md \
1018
- || ! grep -Fq 'spec.expected.json.verification_commands[].cmd' .agents/skills/devlyn:ideate/references/from-spec-mode.md \
1019
- || ! grep -Fq "source for VERIFY's canonical \`spec.solo_headroom_hypothesis\` trigger reason" config/skills/devlyn:ideate/references/spec-template.md \
1020
- || ! grep -Fq "source for VERIFY's canonical \`spec.solo_headroom_hypothesis\` trigger reason" .agents/skills/devlyn:ideate/references/spec-template.md \
1021
- || ! grep -Fq 'spec.expected.json.verification_commands[].cmd' config/skills/devlyn:ideate/references/spec-template.md \
1022
- || ! grep -Fq 'spec.expected.json.verification_commands[].cmd' .agents/skills/devlyn:ideate/references/spec-template.md; then
1023
- bad "ideate solo-headroom hypothesis prompt must match the actionable checker contract"
1024
- else
1025
- ok "ideate solo-headroom hypothesis prompt matches checker contract"
1026
- fi
1027
- if ! grep -Fq 'quick mode must not infer a solo-headroom hypothesis' config/skills/devlyn:ideate/references/elicitation.md \
1028
- || ! grep -Fq 'quick mode must not infer a solo-headroom hypothesis' .agents/skills/devlyn:ideate/references/elicitation.md \
1029
- || ! grep -Fq 'do not infer a solo-headroom hypothesis' config/skills/devlyn:ideate/SKILL.md \
1030
- || ! grep -Fq 'do not infer a solo-headroom hypothesis' .agents/skills/devlyn:ideate/SKILL.md; then
1031
- bad "ideate quick mode must not invent solo-headroom hypotheses"
1032
- else
1033
- ok "ideate quick mode does not invent solo-headroom hypotheses"
1034
- fi
1035
- if ! grep -Fq 'solo ceiling avoidance' config/skills/devlyn:ideate/references/spec-template.md \
1036
- || ! grep -Fq 'solo ceiling avoidance' .agents/skills/devlyn:ideate/references/spec-template.md \
1037
- || ! grep -Fq 'rejected or solo-saturated controls such as `S2`-`S6`' config/skills/devlyn:ideate/references/spec-template.md \
1038
- || ! grep -Fq 'rejected or solo-saturated controls such as `S2`-`S6`' .agents/skills/devlyn:ideate/references/spec-template.md \
1039
- || ! grep -Fq 'Solo ceiling avoidance' config/skills/devlyn:ideate/references/elicitation.md \
1040
- || ! grep -Fq 'Solo ceiling avoidance' .agents/skills/devlyn:ideate/references/elicitation.md \
1041
- || ! grep -Fq 'spec not ready — solo ceiling avoidance required' config/skills/devlyn:ideate/references/elicitation.md \
1042
- || ! grep -Fq 'spec not ready — solo ceiling avoidance required' .agents/skills/devlyn:ideate/references/elicitation.md \
1043
- || ! grep -Fq 'pair-evidence not ready — Pair-candidate headroom is unproven until the spec states solo ceiling avoidance' config/skills/devlyn:ideate/references/from-spec-mode.md \
1044
- || ! grep -Fq 'pair-evidence not ready — Pair-candidate headroom is unproven until the spec states solo ceiling avoidance' .agents/skills/devlyn:ideate/references/from-spec-mode.md \
1045
- || ! grep -Fq 'also do not infer solo ceiling avoidance' config/skills/devlyn:ideate/SKILL.md \
1046
- || ! grep -Fq 'also do not infer solo ceiling avoidance' .agents/skills/devlyn:ideate/SKILL.md \
1047
- || ! grep -Fq 'per-feature Verification must also include a solo ceiling avoidance note' config/skills/devlyn:ideate/references/project-mode.md \
1048
- || ! grep -Fq 'per-feature Verification must also include a solo ceiling avoidance note' .agents/skills/devlyn:ideate/references/project-mode.md; then
1049
- bad "ideate must require solo ceiling avoidance for new unmeasured pair candidates"
1050
- else
1051
- ok "ideate requires solo ceiling avoidance for new unmeasured pair candidates"
1052
- fi
1053
- if ! grep -Fq 'complexity: medium' config/skills/devlyn:ideate/references/spec-template.md \
1054
- || ! grep -Fq 'complexity: medium' .agents/skills/devlyn:ideate/references/spec-template.md \
1055
- || ! grep -Fq 'Complexity signal' config/skills/devlyn:ideate/references/elicitation.md \
1056
- || ! grep -Fq 'Complexity signal' .agents/skills/devlyn:ideate/references/elicitation.md \
1057
- || ! grep -Fq 'downstream VERIFY pair-trigger signal' config/skills/devlyn:ideate/references/elicitation.md \
1058
- || ! grep -Fq 'downstream VERIFY pair-trigger signal' .agents/skills/devlyn:ideate/references/elicitation.md \
1059
- || ! grep -Fq 'complexity=medium default' config/skills/devlyn:ideate/references/from-spec-mode.md \
1060
- || ! grep -Fq 'complexity=medium default' .agents/skills/devlyn:ideate/references/from-spec-mode.md \
1061
- || ! grep -Fq 'supported `complexity` frontmatter' config/skills/devlyn:ideate/SKILL.md \
1062
- || ! grep -Fq 'supported `complexity` frontmatter' .agents/skills/devlyn:ideate/SKILL.md \
1063
- || ! grep -Fq 'supported `complexity` frontmatter' config/skills/devlyn:ideate/references/elicitation.md \
1064
- || ! grep -Fq 'supported `complexity` frontmatter' .agents/skills/devlyn:ideate/references/elicitation.md \
1065
- || ! grep -Fq 'supported `complexity` frontmatter' config/skills/devlyn:resolve/SKILL.md \
1066
- || ! grep -Fq 'supported `complexity` frontmatter' .agents/skills/devlyn:resolve/SKILL.md \
1067
- || ! grep -Fq 'sibling spec `complexity` frontmatter' config/skills/devlyn:ideate/SKILL.md \
1068
- || ! grep -Fq 'sibling spec `complexity` frontmatter' .agents/skills/devlyn:ideate/SKILL.md \
1069
- || ! grep -Fq 'sibling spec `complexity` frontmatter' config/skills/devlyn:resolve/SKILL.md \
1070
- || ! grep -Fq 'sibling spec `complexity` frontmatter' .agents/skills/devlyn:resolve/SKILL.md \
1071
- || ! grep -Fq 'Frontmatter has `id`, `title`, `kind`, `status: planned`, `complexity`' config/skills/devlyn:ideate/SKILL.md \
1072
- || ! grep -Fq 'Frontmatter has `id`, `title`, `kind`, `status: planned`, `complexity`' .agents/skills/devlyn:ideate/SKILL.md; then
1073
- bad "ideate specs must emit complexity frontmatter for resolve pair triggers"
1074
- else
1075
- ok "ideate specs emit complexity frontmatter for resolve pair triggers"
1076
- fi
1077
- if ! grep -Fq 'warning: Verification may need one compound end-to-end scenario before pair-relevant risks are measurable' \
1078
- config/skills/devlyn:ideate/references/from-spec-mode.md \
1079
- || ! grep -Fq 'warning: Verification may need one compound end-to-end scenario before pair-relevant risks are measurable' .agents/skills/devlyn:ideate/references/from-spec-mode.md; then
1080
- bad "ideate from-spec mode must warn when preserved high-risk specs lack compound verification"
1081
- else
1082
- ok "ideate from-spec mode warns on pair-relevant specs with weak verification"
1083
- fi
1084
- if ! grep -Fq 'pair-evidence not ready — Pair-candidate headroom is unproven until the spec states a solo-headroom hypothesis' \
1085
- config/skills/devlyn:ideate/references/from-spec-mode.md \
1086
- || ! grep -Fq 'pair-evidence not ready — Pair-candidate headroom is unproven until the spec states a solo-headroom hypothesis' .agents/skills/devlyn:ideate/references/from-spec-mode.md \
1087
- || ! grep -Fq 'Do not call' config/skills/devlyn:ideate/references/from-spec-mode.md \
1088
- || ! grep -Fq 'Do not call' .agents/skills/devlyn:ideate/references/from-spec-mode.md \
1089
- || ! grep -Fq 'announcement must say `pair-evidence not ready`' config/skills/devlyn:ideate/SKILL.md \
1090
- || ! grep -Fq 'announcement must say `pair-evidence not ready`' .agents/skills/devlyn:ideate/SKILL.md; then
1091
- bad "ideate from-spec mode must warn when pair-candidate specs lack solo-headroom hypothesis"
1092
- else
1093
- ok "ideate from-spec mode warns on missing solo-headroom hypothesis"
1094
- fi
1095
- if ! grep -Fq 'per-feature Verification must' config/skills/devlyn:ideate/references/project-mode.md \
1096
- || ! grep -Fq 'per-feature Verification must' .agents/skills/devlyn:ideate/references/project-mode.md; then
1097
- bad "ideate project mode must require compound verification inside each pair-relevant feature spec"
1098
- else
1099
- ok "ideate project mode keeps compound verification inside pair-relevant feature specs"
1100
- fi
1101
- if ! grep -Fq 'per-feature Verification must include a solo-headroom' config/skills/devlyn:ideate/references/project-mode.md \
1102
- || ! grep -Fq 'per-feature Verification must include a solo-headroom' .agents/skills/devlyn:ideate/references/project-mode.md \
1103
- || grep -rq 'Context or Verification' config/skills/devlyn:ideate/references .agents/skills/devlyn:ideate/references; then
1104
- bad "ideate project mode must keep solo-headroom hypothesis inside each pair-candidate feature spec"
1105
- else
1106
- ok "ideate project mode keeps solo-headroom hypothesis inside each pair-candidate feature spec"
1107
- fi
1108
- if ! grep -Fq 'feature spec must literally contain' config/skills/devlyn:ideate/references/project-mode.md \
1109
- || ! grep -Fq 'feature spec must literally contain' .agents/skills/devlyn:ideate/references/project-mode.md \
1110
- || ! grep -Fq '`solo-headroom hypothesis`, `solo_claude`, `miss`, and a backticked' config/skills/devlyn:ideate/references/project-mode.md \
1111
- || ! grep -Fq '`solo-headroom hypothesis`, `solo_claude`, `miss`, and a backticked' .agents/skills/devlyn:ideate/references/project-mode.md \
1112
- || ! grep -Fq 'behavior a capable' config/skills/devlyn:ideate/references/project-mode.md \
1113
- || ! grep -Fq 'behavior a capable' .agents/skills/devlyn:ideate/references/project-mode.md \
1114
- || ! grep -Fq 'backticked line itself must' config/skills/devlyn:ideate/references/project-mode.md \
1115
- || ! grep -Fq 'backticked line itself must' .agents/skills/devlyn:ideate/references/project-mode.md \
1116
- || ! grep -Fq 'command/observable' config/skills/devlyn:ideate/references/project-mode.md \
1117
- || ! grep -Fq 'command/observable' .agents/skills/devlyn:ideate/references/project-mode.md; then
1118
- bad "ideate project mode solo-headroom prompt must keep the actionable checker contract"
1119
- else
1120
- ok "ideate project mode solo-headroom prompt keeps checker contract"
1121
- fi
1122
-
1123
- if ! grep -Fq 'If the visible spec includes a solo-headroom hypothesis, the first probe must' config/skills/devlyn:resolve/references/phases/probe-derive.md \
1124
- || ! grep -Fq 'If the visible spec includes a solo-headroom hypothesis, the first probe must' .agents/skills/devlyn:resolve/references/phases/probe-derive.md \
1125
- || ! grep -Fq 'When the visible spec includes a solo-headroom hypothesis, the first probe must' config/skills/devlyn:resolve/SKILL.md \
1126
- || ! grep -Fq 'When the visible spec includes a solo-headroom hypothesis, the first probe must' .agents/skills/devlyn:resolve/SKILL.md \
1127
- || ! grep -Fq 'its `cmd` must contain the hypothesis'\''s backticked' config/skills/devlyn:resolve/SKILL.md \
1128
- || ! grep -Fq 'its `cmd` must contain the hypothesis'\''s backticked' .agents/skills/devlyn:resolve/SKILL.md \
1129
- || ! grep -Fq 'its `derived_from` must reference the hypothesis bullet' config/skills/devlyn:resolve/SKILL.md \
1130
- || ! grep -Fq 'its `derived_from` must reference the hypothesis bullet' .agents/skills/devlyn:resolve/SKILL.md; then
1131
- bad "resolve risk-probe prompts must consume solo-headroom hypothesis before pair-evidence work"
1132
- else
1133
- ok "resolve risk-probe prompts consume solo-headroom hypothesis"
1134
- fi
1135
- if ! grep -Fq 'the behavior the spec says `solo_claude` is expected to miss' config/skills/devlyn:resolve/references/phases/probe-derive.md \
1136
- || ! grep -Fq 'the behavior the spec says `solo_claude` is expected to miss' .agents/skills/devlyn:resolve/references/phases/probe-derive.md \
1137
- || ! grep -Fq 'exercise the stated `solo_claude` miss' config/skills/devlyn:resolve/references/phases/probe-derive.md \
1138
- || ! grep -Fq 'exercise the stated `solo_claude` miss' .agents/skills/devlyn:resolve/references/phases/probe-derive.md \
1139
- || ! grep -Fq 'with a `cmd` containing the hypothesis'\''s' config/skills/devlyn:resolve/references/phases/probe-derive.md \
1140
- || ! grep -Fq 'with a `cmd` containing the hypothesis'\''s' .agents/skills/devlyn:resolve/references/phases/probe-derive.md \
1141
- || ! grep -Fq '`derived_from` pointing at the hypothesis' config/skills/devlyn:resolve/references/phases/probe-derive.md \
1142
- || ! grep -Fq '`derived_from` pointing at the hypothesis' .agents/skills/devlyn:resolve/references/phases/probe-derive.md; then
1143
- bad "resolve risk-probe solo-headroom prompt must target the stated solo_claude miss"
1144
- else
1145
- ok "resolve risk-probe solo-headroom prompt targets the stated solo_claude miss"
1146
- fi
1147
- if ! grep -Fq 'If the spec includes a solo-headroom hypothesis, one targeted review must use' \
1148
- config/skills/devlyn:resolve/references/phases/verify.md \
1149
- || ! grep -Fq 'If the spec includes a solo-headroom hypothesis, one targeted review must use' \
1150
- .agents/skills/devlyn:resolve/references/phases/verify.md \
1151
- || ! grep -Fq 'If the spec includes a solo-headroom hypothesis, one targeted review must use' \
1152
- config/skills/devlyn:resolve/SKILL.md \
1153
- || ! grep -Fq 'If the spec includes a solo-headroom hypothesis, one targeted review must use' \
1154
- .agents/skills/devlyn:resolve/SKILL.md \
1155
- || ! grep -Fq 'hypothesis'\''s backticked observable command as the exact anchor' \
1156
- config/skills/devlyn:resolve/SKILL.md \
1157
- || ! grep -Fq 'hypothesis'\''s backticked observable command as the exact anchor' \
1158
- .agents/skills/devlyn:resolve/SKILL.md \
1159
- || ! grep -Fq 'hypothesis'\''s backticked observable command as its exact anchor' \
1160
- config/skills/devlyn:resolve/references/phases/verify.md \
1161
- || ! grep -Fq 'hypothesis'\''s backticked observable command as its exact anchor' \
1162
- .agents/skills/devlyn:resolve/references/phases/verify.md; then
1163
- bad "resolve pair-JUDGE prompts must prioritize solo-headroom hypothesis"
1164
- else
1165
- ok "resolve pair-JUDGE prompts prioritize solo-headroom hypothesis"
1166
- fi
1167
- if ! grep -Fq 'The `--engine` flag does not disable default pairing' config/skills/devlyn:resolve/references/phases/verify.md \
1168
- || ! grep -Fq 'The `--engine` flag does not disable default pairing' .agents/skills/devlyn:resolve/references/phases/verify.md \
1169
- || ! grep -Fq 'the second judge uses the OTHER engine by default when available' config/skills/devlyn:resolve/SKILL.md \
1170
- || ! grep -Fq 'the second judge uses the OTHER engine by default when available' .agents/skills/devlyn:resolve/SKILL.md \
1171
- || ! grep -Fq '_shared/adapters/<name>.md' config/skills/devlyn:resolve/references/phases/verify.md \
1172
- || ! grep -Fq '`## Invocation`' config/skills/devlyn:resolve/references/phases/verify.md \
1173
- || ! grep -Fq '_shared/adapters/<name>.md` `## Invocation' config/skills/devlyn:resolve/SKILL.md \
1174
- || ! grep -Fq '## Invocation' config/skills/_shared/adapters/claude.md \
1175
- || ! grep -Fq '.devlyn/claude-judge.stdout' config/skills/_shared/adapters/claude.md; then
1176
- bad "engine-neutral pair-judge dual declaration (iter-0060) out of sync"
1177
- else
1178
- ok "engine-neutral pair-judge dual declaration (iter-0060) in sync"
1179
- fi
1180
- if ! grep -Fq '`complexity.high`, `complexity.large`' config/skills/devlyn:resolve/SKILL.md \
1181
- || ! grep -Fq '`complexity.high`, `complexity.large`' .agents/skills/devlyn:resolve/SKILL.md \
1182
- || ! grep -Fq '`spec.complexity.high`, `spec.complexity.large`' config/skills/devlyn:resolve/SKILL.md \
1183
- || ! grep -Fq '`spec.complexity.high`, `spec.complexity.large`' .agents/skills/devlyn:resolve/SKILL.md \
1184
- || ! grep -Fq 'Legacy complexity values remain accepted only for archived compatibility' config/skills/devlyn:resolve/references/state-schema.md \
1185
- || ! grep -Fq 'Legacy complexity values remain accepted only for archived compatibility' .agents/skills/devlyn:resolve/references/state-schema.md \
1186
- || ! grep -Fq '`complexity.high`, `complexity.large`, `spec.complexity.high`, `spec.complexity.large`' config/skills/devlyn:resolve/references/state-schema.md \
1187
- || ! grep -Fq '`complexity.high`, `complexity.large`, `spec.complexity.high`, `spec.complexity.large`' .agents/skills/devlyn:resolve/references/state-schema.md \
1188
- || ! grep -Fq '`complexity.large`, `spec.complexity.high`, `spec.complexity.large`,' config/skills/devlyn:resolve/references/phases/verify.md \
1189
- || ! grep -Fq '`complexity.large`, `spec.complexity.high`, `spec.complexity.large`,' .agents/skills/devlyn:resolve/references/phases/verify.md \
1190
- || ! grep -Fq '`spec.solo_headroom_hypothesis`, `risk.high`, `risk_probes.enabled`,' config/skills/devlyn:resolve/references/phases/verify.md \
1191
- || ! grep -Fq '`spec.solo_headroom_hypothesis`, `risk.high`, `risk_probes.enabled`,' .agents/skills/devlyn:resolve/references/phases/verify.md; then
1192
- bad "resolve VERIFY docs must distinguish current large complexity, legacy high state, and legacy large spec compatibility"
1193
- else
1194
- ok "resolve VERIFY docs distinguish current large complexity, legacy high state, and legacy large spec compatibility"
1195
- fi
1196
- if ! grep -Fq 'def spec_has_solo_headroom_hypothesis' config/skills/_shared/verify-merge-findings.py \
1197
- || ! grep -Fq 'spec.solo_headroom_hypothesis' config/skills/_shared/verify-merge-findings.py \
1198
- || ! grep -Fq 'and "solo_claude" in lower' config/skills/_shared/verify-merge-findings.py \
1199
- || ! grep -Fq 'and "miss" in lower' config/skills/_shared/verify-merge-findings.py \
1200
- || ! grep -Fq 'OBSERVABLE_COMMAND_MARKERS = ("command", "observable", "expose")' config/skills/_shared/verify-merge-findings.py \
1201
- || ! grep -Fq 'def is_command_like_backtick' config/skills/_shared/verify-merge-findings.py \
1202
- || ! grep -Fq 'for key in ("spec_path", "criteria_path")' config/skills/_shared/verify-merge-findings.py \
1203
- || ! grep -Fq 'has_backticked_observable_command(text)' config/skills/_shared/verify-merge-findings.py \
1204
- || ! grep -Fq 'Observable command: `node check.js` exposes behavior' config/skills/_shared/verify-merge-findings.py \
1205
- || ! grep -Fq 'observable `SOLO_CLAUDE` exposes the miss' config/skills/_shared/verify-merge-findings.py \
1206
- || ! grep -Fq 'observable `priority rollback` exposes the miss' config/skills/_shared/verify-merge-findings.py \
1207
- || ! grep -Fq 'implementation token `rollback`' config/skills/_shared/verify-merge-findings.py \
1208
- || ! grep -Fq '`SOLO_CLAUDE` should miss' config/skills/_shared/verify-merge-findings.py \
1209
- || ! grep -Fq '{"source": {"criteria_path": str(criteria_path)}}' config/skills/_shared/verify-merge-findings.py \
1210
- || ! grep -Fq 'spec_has_solo_headroom_hypothesis(' config/skills/_shared/verify-merge-findings.py \
1211
- || ! grep -Fq ') is False' config/skills/_shared/verify-merge-findings.py \
1212
- || ! grep -Fq ') is True' config/skills/_shared/verify-merge-findings.py \
1213
- || ! grep -Fq 'def spec_has_solo_headroom_hypothesis' .agents/skills/_shared/verify-merge-findings.py \
1214
- || ! grep -Fq 'spec.solo_headroom_hypothesis' .agents/skills/_shared/verify-merge-findings.py \
1215
- || ! grep -Fq 'and "solo_claude" in lower' .agents/skills/_shared/verify-merge-findings.py \
1216
- || ! grep -Fq 'and "miss" in lower' .agents/skills/_shared/verify-merge-findings.py \
1217
- || ! grep -Fq 'OBSERVABLE_COMMAND_MARKERS = ("command", "observable", "expose")' .agents/skills/_shared/verify-merge-findings.py \
1218
- || ! grep -Fq 'def is_command_like_backtick' .agents/skills/_shared/verify-merge-findings.py \
1219
- || ! grep -Fq 'for key in ("spec_path", "criteria_path")' .agents/skills/_shared/verify-merge-findings.py \
1220
- || ! grep -Fq 'has_backticked_observable_command(text)' .agents/skills/_shared/verify-merge-findings.py \
1221
- || ! grep -Fq 'Observable command: `node check.js` exposes behavior' .agents/skills/_shared/verify-merge-findings.py \
1222
- || ! grep -Fq 'observable `SOLO_CLAUDE` exposes the miss' .agents/skills/_shared/verify-merge-findings.py \
1223
- || ! grep -Fq 'observable `priority rollback` exposes the miss' .agents/skills/_shared/verify-merge-findings.py \
1224
- || ! grep -Fq 'implementation token `rollback`' .agents/skills/_shared/verify-merge-findings.py \
1225
- || ! grep -Fq '`SOLO_CLAUDE` should miss' .agents/skills/_shared/verify-merge-findings.py \
1226
- || ! grep -Fq '{"source": {"criteria_path": str(criteria_path)}}' .agents/skills/_shared/verify-merge-findings.py \
1227
- || ! grep -Fq 'spec_has_solo_headroom_hypothesis(' .agents/skills/_shared/verify-merge-findings.py \
1228
- || ! grep -Fq ') is False' .agents/skills/_shared/verify-merge-findings.py \
1229
- || ! grep -Fq ') is True' .agents/skills/_shared/verify-merge-findings.py \
1230
- || ! grep -Fq '`spec.solo_headroom_hypothesis`' config/skills/devlyn:resolve/references/phases/verify.md \
1231
- || ! grep -Fq '`spec.solo_headroom_hypothesis`' .agents/skills/devlyn:resolve/references/phases/verify.md \
1232
- || ! grep -Fq '`spec.solo_headroom_hypothesis`' config/skills/devlyn:resolve/references/state-schema.md \
1233
- || ! grep -Fq '`spec.solo_headroom_hypothesis`' .agents/skills/devlyn:resolve/references/state-schema.md \
1234
- || ! grep -Fq 'same actionable solo-headroom hypothesis is a VERIFY pair-trigger reason' config/skills/devlyn:resolve/SKILL.md \
1235
- || ! grep -Fq 'same actionable solo-headroom hypothesis is a VERIFY pair-trigger reason' .agents/skills/devlyn:resolve/SKILL.md; then
1236
- bad "resolve VERIFY pair trigger must include actionable solo-headroom hypothesis specs"
1237
- else
1238
- ok "resolve VERIFY pair trigger includes actionable solo-headroom hypothesis specs"
1239
- fi
1240
- if ! grep -Fq 'pair_evidence_intent' config/skills/devlyn:resolve/references/free-form-mode.md \
1241
- || ! grep -Fq 'pair_evidence_intent' .agents/skills/devlyn:resolve/references/free-form-mode.md \
1242
- || ! grep -Fq 'has_actionable_solo_headroom' config/skills/devlyn:resolve/references/free-form-mode.md \
1243
- || ! grep -Fq 'has_actionable_solo_headroom' .agents/skills/devlyn:resolve/references/free-form-mode.md \
1244
- || ! grep -Fq 'state.source.type = "generated"' config/skills/devlyn:resolve/SKILL.md \
1245
- || ! grep -Fq 'state.source.type = "generated"' .agents/skills/devlyn:resolve/SKILL.md \
1246
- || ! grep -Fq 'state.source.criteria_path = ".devlyn/criteria.generated.md"' config/skills/devlyn:resolve/references/free-form-mode.md \
1247
- || ! grep -Fq 'state.source.criteria_path = ".devlyn/criteria.generated.md"' .agents/skills/devlyn:resolve/references/free-form-mode.md \
1248
- || ! grep -Fq 'state.source.criteria_sha256' config/skills/devlyn:resolve/references/free-form-mode.md \
1249
- || ! grep -Fq 'state.source.criteria_sha256' .agents/skills/devlyn:resolve/references/free-form-mode.md \
1250
- || ! grep -Fq 'state.source.criteria_sha256` for generated free-form mode' config/skills/devlyn:resolve/references/phases/verify.md \
1251
- || ! grep -Fq 'state.source.criteria_sha256` for generated free-form mode' .agents/skills/devlyn:resolve/references/phases/verify.md \
1252
- || ! grep -Fq 'Free-form sets `type: "generated"`' config/skills/devlyn:resolve/references/state-schema.md \
1253
- || ! grep -Fq 'Free-form sets `type: "generated"`' .agents/skills/devlyn:resolve/references/state-schema.md \
1254
- || ! grep -Fq 'backticked observable command line that itself contains `miss`' config/skills/devlyn:resolve/references/free-form-mode.md \
1255
- || ! grep -Fq 'backticked observable command line that itself contains `miss`' .agents/skills/devlyn:resolve/references/free-form-mode.md \
1256
- || ! grep -Fq 'BLOCKED:solo-headroom-hypothesis-required' config/skills/devlyn:resolve/references/free-form-mode.md \
1257
- || ! grep -Fq 'BLOCKED:solo-headroom-hypothesis-required' .agents/skills/devlyn:resolve/references/free-form-mode.md \
1258
- || ! grep -Fq 'pair-evidence intent without an actionable solo-headroom hypothesis must halt' config/skills/devlyn:resolve/SKILL.md \
1259
- || ! grep -Fq 'pair-evidence intent without an actionable solo-headroom hypothesis must halt' .agents/skills/devlyn:resolve/SKILL.md \
1260
- || ! grep -Fq 'preserve that literal hypothesis in `.devlyn/criteria.generated.md`' config/skills/devlyn:resolve/references/free-form-mode.md \
1261
- || ! grep -Fq 'preserve that literal hypothesis in `.devlyn/criteria.generated.md`' .agents/skills/devlyn:resolve/references/free-form-mode.md \
1262
- || ! grep -Fq 'emit the canonical `spec.solo_headroom_hypothesis` pair trigger reason' config/skills/devlyn:resolve/references/free-form-mode.md \
1263
- || ! grep -Fq 'emit the canonical `spec.solo_headroom_hypothesis` pair trigger reason' .agents/skills/devlyn:resolve/references/free-form-mode.md \
1264
- || ! grep -Fq '/devlyn:ideate` guidance for `BLOCKED:solo-headroom-hypothesis-required`' config/skills/devlyn:resolve/references/state-schema.md \
1265
- || ! grep -Fq '/devlyn:ideate` guidance for `BLOCKED:solo-headroom-hypothesis-required`' .agents/skills/devlyn:resolve/references/state-schema.md \
1266
- || ! grep -Fq '/devlyn:ideate` guidance after `BLOCKED:solo-headroom-hypothesis-required`' config/skills/devlyn:resolve/SKILL.md \
1267
- || ! grep -Fq '/devlyn:ideate` guidance after `BLOCKED:solo-headroom-hypothesis-required`' .agents/skills/devlyn:resolve/SKILL.md \
1268
- || ! grep -Fq 'Free-form goals that ask for benchmark evidence, pair-evidence, risk-probe' README.md \
1269
- || ! grep -Fq '`/devlyn:resolve` stops with `BLOCKED:solo-headroom-hypothesis-required`' README.md \
1270
- || ! grep -Fq 'points you to `/devlyn:ideate` instead of inventing a weak hypothesis' README.md; then
1271
- bad "resolve free-form mode must block pair-evidence goals without actionable solo-headroom hypothesis"
1272
- else
1273
- ok "resolve free-form mode blocks pair-evidence goals without actionable solo-headroom hypothesis"
1274
- fi
1275
- if ! grep -Fq 'unmeasured_pair_candidate_intent' config/skills/devlyn:resolve/references/free-form-mode.md \
1276
- || ! grep -Fq 'unmeasured_pair_candidate_intent' .agents/skills/devlyn:resolve/references/free-form-mode.md \
1277
- || ! grep -Fq 'has_solo_ceiling_avoidance' config/skills/devlyn:resolve/references/free-form-mode.md \
1278
- || ! grep -Fq 'has_solo_ceiling_avoidance' .agents/skills/devlyn:resolve/references/free-form-mode.md \
1279
- || ! grep -Fq 'BLOCKED:solo-ceiling-avoidance-required' config/skills/devlyn:resolve/references/free-form-mode.md \
1280
- || ! grep -Fq 'BLOCKED:solo-ceiling-avoidance-required' .agents/skills/devlyn:resolve/references/free-form-mode.md \
1281
- || ! grep -Fq 'unmeasured pair-candidate intent without solo ceiling avoidance must halt' config/skills/devlyn:resolve/SKILL.md \
1282
- || ! grep -Fq 'unmeasured pair-candidate intent without solo ceiling avoidance must halt' .agents/skills/devlyn:resolve/SKILL.md \
1283
- || ! grep -Fq '/devlyn:ideate` guidance after `BLOCKED:solo-ceiling-avoidance-required`' config/skills/devlyn:resolve/SKILL.md \
1284
- || ! grep -Fq '/devlyn:ideate` guidance after `BLOCKED:solo-ceiling-avoidance-required`' .agents/skills/devlyn:resolve/SKILL.md \
1285
- || ! grep -Fq '/devlyn:ideate` guidance for `BLOCKED:solo-ceiling-avoidance-required`' config/skills/devlyn:resolve/references/state-schema.md \
1286
- || ! grep -Fq '/devlyn:ideate` guidance for `BLOCKED:solo-ceiling-avoidance-required`' .agents/skills/devlyn:resolve/references/state-schema.md \
1287
- || ! grep -Fq '`/devlyn:resolve` stops with `BLOCKED:solo-ceiling-avoidance-required`' README.md; then
1288
- bad "resolve free-form mode must block new unmeasured pair candidates without solo ceiling avoidance"
1289
- else
1290
- ok "resolve free-form mode blocks new unmeasured pair candidates without solo ceiling avoidance"
1291
- fi
1292
-
1293
- section "Check 6g: resolve consumes sibling spec.expected.json"
1294
- sibling_consume_missing=0
1295
- for file in \
1296
- config/skills/devlyn:resolve/SKILL.md \
1297
- config/skills/devlyn:resolve/references/phases/build-gate.md \
1298
- config/skills/devlyn:resolve/references/phases/verify.md
1299
- do
1300
- if ! grep -Fq 'sibling `spec.expected.json`' "$file"; then
1301
- bad "$file — missing sibling spec.expected.json consumption contract"
1302
- sibling_consume_missing=1
1303
- fi
1304
- done
1305
- for pattern in \
1306
- 'def stage_from_expected' \
1307
- 'stage_from_expected(' \
1308
- 'contract_found, _staged, expected_error, expected_path' \
1309
- 'def expected_contract_findings' \
1310
- 'correctness.forbidden-pattern' \
1311
- 'scope.max-deps-added-exceeded' \
1312
- 'SPEC_VERIFY_FINDINGS_FILE'
1313
- do
1314
- if ! grep -Fq "$pattern" config/skills/_shared/spec-verify-check.py; then
1315
- bad "spec-verify-check.py missing sibling expected staging implementation: $pattern"
1316
- sibling_consume_missing=1
1317
- fi
1318
- done
1319
- if [ $sibling_consume_missing -eq 0 ]; then
1320
- ok "resolve self-stages and mechanically checks sibling spec.expected.json"
1321
- fi
1322
-
1323
- section "Check 6i: VERIFY mechanical findings are merge-visible"
1324
- verify_mech_missing=0
1325
- for pattern in \
1326
- 'SPEC_VERIFY_PHASE=verify_mechanical' \
1327
- 'SPEC_VERIFY_FINDINGS_FILE=verify-mechanical.findings.jsonl' \
1328
- 'SPEC_VERIFY_FINDING_PREFIX=VERIFY-MECH'
1329
- do
1330
- if ! grep -Fq "$pattern" config/skills/devlyn:resolve/SKILL.md \
1331
- || ! grep -Fq "$pattern" config/skills/devlyn:resolve/references/phases/verify.md \
1332
- || ! grep -Fq "$pattern" config/skills/_shared/spec-verify-check.py; then
1333
- bad "VERIFY mechanical output contract missing: $pattern"
1334
- verify_mech_missing=1
1335
- fi
1336
- done
1337
- if ! grep -Fq '("mechanical", "verify-mechanical.findings.jsonl")' \
1338
- config/skills/_shared/verify-merge-findings.py; then
1339
- bad "verify-merge-findings.py does not consume verify-mechanical.findings.jsonl"
1340
- verify_mech_missing=1
1341
- fi
1342
- if grep -Fq 'All paths emit a CRITICAL finding to' config/skills/_shared/spec-verify-check.py; then
1343
- bad "spec-verify-check.py has stale single-output CRITICAL finding wording"
1344
- verify_mech_missing=1
1345
- fi
1346
- if [ $verify_mech_missing -eq 0 ]; then
1347
- ok "spec-verify VERIFY output routes into verify-merge-findings.py"
1348
- fi
1349
-
1350
- section "Check 6i1: Verification provenance contract is wired"
1351
- verification_provenance_missing=0
1352
- for file in \
1353
- config/skills/devlyn:resolve/SKILL.md \
1354
- .agents/skills/devlyn:resolve/SKILL.md
1355
- do
1356
- if ! grep -Fq 'orchestrator commands with no separate model' "$file" \
1357
- || ! grep -Fq 'execution_kind: "orchestrator_commands"' "$file" \
1358
- || grep -Fq 'resolved BUILD_GATE engine through the fresh-worker route' "$file" \
1359
- || grep -Fq 'Codex-routed IMPLEMENT, BUILD_GATE, or CLEANUP spawn' "$file" \
1360
- || ! grep -Fq 'BLOCKED:build-env-underprovisioned' "$file" \
1361
- || ! grep -Fq '.devlyn/<primary-engine>-judge.stdout' "$file" \
1362
- || ! grep -Fq '.devlyn/<other-engine>-judge.stdout' "$file" \
1363
- || ! grep -Fq 'state-bound process-evidence manifest/raw stream' "$file"; then
1364
- bad "$file — orchestrator provenance/capability/archive contract missing"
1365
- verification_provenance_missing=1
1366
- fi
1367
- done
1368
- for file in \
1369
- config/skills/devlyn:resolve/references/phases/verify.md \
1370
- .agents/skills/devlyn:resolve/references/phases/verify.md
1371
- do
1372
- if ! grep -Fq 'executable verification belongs exclusively to MECHANICAL' "$file" \
1373
- || ! grep -Fq 'native read/search tools or non-mutating shell commands' "$file" \
1374
- || ! grep -Fq 'JUDGE does not execute literal verification, lint, test, build, risk-probe, or' "$file" \
1375
- || ! grep -Fq 'newly invented interaction commands.' "$file" \
1376
- || ! grep -Fq '.devlyn/<primary-engine>-judge.stdout' "$file" \
1377
- || ! grep -Fq '.devlyn/<other-engine>-judge.stdout' "$file" \
1378
- || ! grep -Fq 'process-evidence manifest and raw stdout/stderr streams' "$file"; then
1379
- bad "$file — JUDGE sealed-evidence contract missing"
1380
- verification_provenance_missing=1
1381
- fi
1382
- if grep -Fq 'execute at least one combined adversarial check' "$file" \
1383
- || grep -Fq 'Execute at most two targeted probes' "$file" \
1384
- || grep -Fq "repo's CLI/API/test runner" "$file"; then
1385
- bad "$file — stale JUDGE command-execution instruction remains"
1386
- verification_provenance_missing=1
1387
- fi
1388
- done
1389
- for file in \
1390
- config/skills/devlyn:resolve/SKILL.md \
1391
- .agents/skills/devlyn:resolve/SKILL.md \
1392
- config/skills/devlyn:resolve/references/phases/verify.md \
1393
- .agents/skills/devlyn:resolve/references/phases/verify.md
1394
- do
1395
- if grep -Eiq 'executes? no commands|no code-mutation or command-execution tools|do not receive command-execution tools' "$file"; then
1396
- bad "$file — blanket JUDGE inspection prohibition remains"
1397
- verification_provenance_missing=1
1398
- fi
1399
- done
1400
- if [ $verification_provenance_missing -eq 0 ]; then
1401
- ok "BUILD_GATE, sealed JUDGE inputs, canonical captures, and archive flow are wired"
1402
- fi
1403
-
1404
- section "Check 6j: VERIFY default pair dispatches concurrently"
1405
- pair_trigger_order_missing=0
1406
- for file in \
1407
- config/skills/devlyn:resolve/SKILL.md \
1408
- .agents/skills/devlyn:resolve/SKILL.md
1409
- do
1410
- if ! grep -Fq 'Verify dual-judge is default-when-available' "$file" \
1411
- || ! grep -Fq 'persist `pair_trigger` before spawn with `pair.default`' "$file" \
1412
- || ! grep -Fq 'dispatch both judges concurrently against the same frozen diff' "$file" \
1413
- || ! grep -Fq 'A primary blocker never cancels the pair judge' "$file" \
1414
- || ! grep -Fq 'run the same two required judges sequentially' "$file" \
1415
- || ! grep -Fq 'Eligible schema-v3 state must include `pair.default`' "$file" \
1416
- || ! grep -Fq '`primary_judge_blocker` is parser-recognized only for archived v2.0 replay; new runs never write it' "$file"; then
1417
- bad "$file — VERIFY default pair rule missing."
1418
- pair_trigger_order_missing=1
1419
- fi
1420
- done
1421
- for file in \
1422
- config/skills/devlyn:resolve/references/phases/verify.md \
1423
- .agents/skills/devlyn:resolve/references/phases/verify.md
1424
- do
1425
- if ! grep -Fq 'Pair-mode (default when OTHER engine is available)' "$file" \
1426
- || ! grep -Fq 'with `pair.default` plus every applicable' "$file" \
1427
- || ! grep -Fq 'dispatch the primary and pair JUDGEs concurrently against the same frozen' "$file" \
1428
- || ! grep -Fq 'A primary blocker does not cancel the pair-JUDGE' "$file" \
1429
- || ! grep -Fq 'runs the same two required judges' "$file" \
1430
- || ! grep -Fq 'Eligible schema-v3 state must contain `pair.default`' "$file" \
1431
- || ! grep -Fq 'New runs never' "$file"; then
1432
- bad "$file — VERIFY default pair dispatch missing"
1433
- pair_trigger_order_missing=1
1434
- fi
1435
- done
1436
- for file in \
1437
- config/skills/devlyn:resolve/references/state-schema.md \
1438
- .agents/skills/devlyn:resolve/references/state-schema.md
1439
- do
1440
- if ! grep -Fq '"version": "3.0"' "$file" \
1441
- || ! grep -Fq 'pair.default` dispatches both judges concurrently' "$file" \
1442
- || ! grep -Fq 'runs the same two required judges sequentially' "$file" \
1443
- || ! grep -Fq 'a primary blocker never skips the pair' "$file" \
1444
- || ! grep -Fq 'Schema v3.0 eligible state requires `pair.default`' "$file" \
1445
- || ! grep -Fq 'archived v2.0 replay' "$file" \
1446
- || ! grep -Fq 'Missing, contradictory, incomplete, or unknown trigger state BLOCKs VERIFY' "$file"; then
1447
- bad "$file — schema-v3 default pair contract missing"
1448
- pair_trigger_order_missing=1
1449
- fi
1450
- done
1451
- if [ $pair_trigger_order_missing -eq 0 ]; then
1452
- ok "VERIFY default pair dispatches concurrently"
1453
- fi
1454
-
1455
- section "Check 6k: PLAN route is orchestrator-fixed"
1456
- plan_route_ok=1
1457
- for plan_route_file in "config/skills/devlyn:resolve/SKILL.md" CLAUDE.md AGENTS.md \
1458
- "config/skills/devlyn:engines/SKILL.md" ".agents/skills/devlyn:resolve/SKILL.md" \
1459
- ".agents/skills/devlyn:engines/SKILL.md"; do
1460
- [ -f "$plan_route_file" ] || plan_route_ok=0
1461
- done
1462
- sed -n '/^<engine_routing>$/,/^<\/engine_routing>$/p' "config/skills/devlyn:resolve/SKILL.md" \
1463
- | grep -Fq 'PLAN is orchestrator-fixed and never inherits `--engine`, an executor pin, or `state.engine`' || plan_route_ok=0
1464
- sed -n '/^## PHASE 1: PLAN$/,/^## PHASE 1.5/p' "config/skills/devlyn:resolve/SKILL.md" \
1465
- | grep -Fq 'PLAN is orchestrator-fixed and never inherits `--engine`, an executor pin, or `state.engine`' || plan_route_ok=0
1466
- grep -Fq 'PLAN is orchestrator-fixed and never inherits `--engine` or an executor pin' CLAUDE.md || plan_route_ok=0
1467
- grep -Fq 'PLAN is orchestrator-fixed and never inherits `--engine` or the executor pin' AGENTS.md || plan_route_ok=0
1468
- grep -Fq 'PLAN is orchestrator-fixed and never follows the pin' config/skills/devlyn:engines/SKILL.md || plan_route_ok=0
1469
- if grep -En '([Ee]xecutor|Default engine: Claude for)[^[:cntrl:]]{0,80}PLAN[[:space:]]*/[[:space:]]*IMPLEMENT([[:space:]]*/[[:space:]]*BUILD_GATE)?[[:space:]]*/[[:space:]]*CLEANUP' \
1470
- config/skills/devlyn:resolve/SKILL.md \
1471
- CLAUDE.md \
1472
- AGENTS.md \
1473
- README.md \
1474
- config/skills/devlyn:engines/SKILL.md \
1475
- .agents/skills/devlyn:resolve/SKILL.md \
1476
- .agents/skills/devlyn:engines/SKILL.md \
1477
- >/dev/null 2>&1; then
1478
- plan_route_ok=0
1479
- fi
1480
- if [ $plan_route_ok -eq 1 ]; then
1481
- ok "PLAN is orchestrator-fixed: exact declarations locked on all surfaces, no executor-role enumeration"
1482
- else
1483
- bad "PLAN route contract violated: a locked orchestrator-fixed declaration is missing or an executor-role enumeration re-teaches PLAN"
1484
- fi
1485
-
1486
- section "Check 6h: No undocumented spec.expected.json.browser_flows field"
1487
- browser_flow_refs=$(grep -RInF 'spec.expected.json.browser_flows' \
1488
- config/skills README.md bin/ package.json 2>/dev/null || true)
1489
- if [ -z "$browser_flow_refs" ]; then
1490
- ok "active docs do not advertise unsupported browser_flows schema field"
1491
- else
1492
- while IFS= read -r f; do bad "$f"; done <<< "$browser_flow_refs"
1493
- fi
1494
-
1495
- section "Check 6e: All-or-nothing probes prove mutable rollback"
1496
- probe_doc="config/skills/devlyn:resolve/references/phases/probe-derive.md"
1497
- if grep -Fq "pre-rejected by a whole-order availability shortcut" "$probe_doc" \
1498
- && grep -Fq "must allocate a scarce" "$probe_doc" \
1499
- && grep -Fq "must request the same scarce first-line SKU" "$probe_doc"; then
1500
- ok "all-or-nothing probe contract preserves mutable rollback evidence"
1501
- else
1502
- bad "$probe_doc — missing mutable rollback probe contract"
1503
- fi
1504
-
1505
- # ---------------------------------------------------------------------------
1506
- # 8. CRITIC security sub-pass must be native, not Dual.
1507
- # Catches the specific drift where a section updates but a cross-reference doesn't.
1508
- # ---------------------------------------------------------------------------
1509
- section "Check 8: CRITIC security is native (no stale Dual references)"
1510
- # Match only the concrete bad patterns used when CRITIC security was routed to
1511
- # Dual (v3.4 and earlier):
1512
- # 1. Markdown table cell `| **Dual** |`
1513
- # 2. Prose `Dual (Claude + Codex parallel, merged)`
1514
- # Retrospective mentions like "drops the Dual-model token cost" are fine.
1515
- offenders=$(grep -RInE '\|\s*\*\*Dual\*\*\s*\||Dual\s*\(Claude\s*\+\s*Codex' \
1516
- config/skills 2>/dev/null \
1517
- | grep -v 'roadmap-archival-workspace/' \
1518
- | grep -v 'devlyn:auto-resolve-workspace/' \
1519
- | grep -v 'devlyn:ideate-workspace/' \
1520
- | grep -v 'preflight-workspace/' \
1521
- || true)
1522
- if [ -z "$offenders" ]; then
1523
- ok "CRITIC security uses native (no Dual stragglers)"
1524
- else
1525
- while IFS= read -r f; do bad "$f"; done <<< "$offenders"
1526
- fi
1527
-
1528
- # ---------------------------------------------------------------------------
1529
- # 10. No raw `codex exec` invocation in skill prompts (iter-0010).
1530
- # iter-0009 wrapper + iter-0010 production rollout require every Codex
1531
- # invocation in skill SKILL.md / references to use codex-monitored.sh.
1532
- # Raw `codex exec ...` in a prompt re-introduces the iter-0008 byte-watchdog
1533
- # starvation: orchestrator pattern-primes from the doc and emits the raw
1534
- # shape, which can collapse into `... | tail -200` and starve the outer API
1535
- # stream. Descriptive phrases like "passes args through to `codex exec`
1536
- # verbatim" are allowed — only invocation-shaped uses are forbidden.
1537
- #
1538
- # Pattern: `codex exec[[:space:]]+\S` — catches any invocation shape
1539
- # (whitespace then a non-space character after `exec`). Passes backtick-
1540
- # closed descriptive prose like `` `codex exec` `` because the closing
1541
- # backtick is non-whitespace adjacent to `exec`, not whitespace.
1542
- # Concrete shapes caught:
1543
- # - single-line flag: `codex exec -C ...`
1544
- # - resume form: `codex exec resume --last`
1545
- # - multi-line cont.: `codex exec \` (space + `\` at EOL)
1546
- # - quoted prompt: `codex exec "prompt"` ← iter-0011
1547
- # - variable expansion: `codex exec $PROMPT` ← iter-0011
1548
- # - literal token: `codex exec prompt` ← iter-0011
1549
- # Excludes: _shared/codex-config.md (canonical doc may discuss the rule
1550
- # itself), workspace/, archive snapshots.
1551
- # ---------------------------------------------------------------------------
1552
- section "Check 10: No raw codex exec invocation in skill prompts"
1553
- offenders=$(grep -RInE 'codex exec[[:space:]]+[^[:space:]]' \
1554
- config/skills 2>/dev/null \
1555
- | grep -v 'config/skills/_shared/codex-config.md' \
1556
- | grep -v 'config/skills/_shared/codex-monitored.sh' \
1557
- | grep -v 'roadmap-archival-workspace/' \
1558
- | grep -v 'devlyn:auto-resolve-workspace/' \
1559
- | grep -v 'devlyn:ideate-workspace/' \
1560
- | grep -v 'preflight-workspace/' \
1561
- || true)
1562
- if [ -z "$offenders" ]; then
1563
- ok "no raw codex exec invocations in skill prompts (wrapper-form everywhere)"
1564
- else
1565
- while IFS= read -r f; do bad "$f"; done <<< "$offenders"
1566
- fi
1567
-
1568
- # ---------------------------------------------------------------------------
1569
- # 10a. Bounded Codex probe/judge calls must run isolated.
1570
- # Pair/risk-probe paths are measured read-only judges, not implementation
1571
- # phases. They must not inherit user config, AGENTS.md, hooks, or other
1572
- # local rules that can add hidden context or transcript side effects.
1573
- # The wrapper owns the flag expansion; skill docs own requiring
1574
- # CODEX_MONITORED_ISOLATED=1 for probe-derive and pair-JUDGE.
1575
- # ---------------------------------------------------------------------------
1576
- section "Check 10a0: Codex primary VERIFY budget is bounded and mirrored"
1577
- primary_verify_missing=0
1578
- primary_verify_route='DEVLYN_CODEX_PROMPT_FILE="<primary-prompt-file>" CODEX_MONITORED_ISOLATED=1 CODEX_MONITORED_TIMEOUT_SEC=600 bash "$CODEX_MONITORED_PATH" -C "$PWD" -s read-only -c model_reasoning_effort=high - >.devlyn/codex-judge.stdout 2>.devlyn/codex-judge.stderr'
1579
- for file in \
1580
- config/skills/devlyn:resolve/SKILL.md \
1581
- .agents/skills/devlyn:resolve/SKILL.md \
1582
- config/skills/devlyn:resolve/references/phases/verify.md \
1583
- .agents/skills/devlyn:resolve/references/phases/verify.md
1584
- do
1585
- if { ! grep -Fxq "$primary_verify_route" "$file" \
1586
- && ! grep -Fq "\`$primary_verify_route\`" "$file"; } \
1587
- || ! grep -Fq '.devlyn/verify.primary.timeout.json' "$file" \
1588
- || ! grep -Fq 'pair-style `TIMEOUT`' "$file"; then
1589
- bad "$file — Codex primary-JUDGE route/timeout authority missing"
1590
- primary_verify_missing=1
1591
- fi
1592
- done
1593
- for file in \
1594
- config/skills/devlyn:resolve/references/phases/verify.md \
1595
- .agents/skills/devlyn:resolve/references/phases/verify.md
1596
- do
1597
- for needle in \
1598
- 'one broad pass over the' \
1599
- 'It then makes one' \
1600
- 'targeted interaction pass over the clauses the broad pass left unresolved.' \
1601
- 'Before any third pass' \
1602
- 'verdict-binding BLOCKED coverage finding'
1603
- do
1604
- if ! grep -Fq "$needle" "$file"; then
1605
- bad "$file — bounded primary review instruction missing: $needle"
1606
- primary_verify_missing=1
1607
- fi
1608
- done
1609
- done
1610
- for file in \
1611
- config/skills/_shared/adapters/codex.md \
1612
- .agents/skills/_shared/adapters/codex.md
1613
- do
1614
- for needle in \
1615
- 'batch related reads' \
1616
- 'sealed parity proves' \
1617
- 'Open self-test bodies or MECHANICAL raw streams only for a'
1618
- do
1619
- if ! grep -Fq "$needle" "$file"; then
1620
- bad "$file — Codex bounded retrieval tactic missing: $needle"
1621
- primary_verify_missing=1
1622
- fi
1623
- done
1624
- done
1625
- for file in \
1626
- config/skills/_shared/verify-merge-findings.py \
1627
- .agents/skills/_shared/verify-merge-findings.py \
1628
- config/skills/_shared/archive_run.py \
1629
- .agents/skills/_shared/archive_run.py \
1630
- config/skills/_shared/resolve-bootstrap.py \
1631
- .agents/skills/_shared/resolve-bootstrap.py
1632
- do
1633
- if ! grep -Fq 'verify.primary.timeout.json' "$file"; then
1634
- bad "$file — primary timeout deterministic/archive contract missing"
1635
- primary_verify_missing=1
1636
- fi
1637
- done
1638
- if [ $primary_verify_missing -eq 0 ]; then
1639
- ok "Codex primary VERIFY is bounded, fail-closed, archived, and mirrored"
1640
- fi
1641
-
1642
- section "Check 10a: Bounded Codex calls use isolated wrapper mode"
1643
- isolation_missing=0
1644
- for needle in \
1645
- 'CODEX_MONITORED_ISOLATED=1 bash "$CODEX_MONITORED_PATH"' \
1646
- 'CODEX_MONITORED_ISOLATED=1 CODEX_MONITORED_TIMEOUT_SEC=600` and `-c model_reasoning_effort=medium' \
1647
- 'CODEX_MONITORED_PATH="$DEVLYN_SHARED_DIR/codex-monitored.sh"'
1648
- do
1649
- if ! grep -RInF "$needle" config/skills >/dev/null 2>&1; then
1650
- bad "missing isolated Codex invocation contract: $needle"
1651
- isolation_missing=1
1652
- fi
1653
- done
1654
- for flag in \
1655
- '--ignore-user-config' \
1656
- '--ignore-rules' \
1657
- '--ephemeral' \
1658
- '--disable codex_hooks' \
1659
- '--disable hooks'
1660
- do
1661
- if ! grep -F -- "$flag" config/skills/_shared/codex-monitored.sh >/dev/null 2>&1; then
1662
- bad "codex-monitored.sh missing isolation flag expansion: $flag"
1663
- isolation_missing=1
1664
- fi
1665
- done
1666
- for helper in require_positive_int require_nonnegative_int; do
1667
- if ! grep -F "$helper" config/skills/_shared/codex-monitored.sh >/dev/null 2>&1; then
1668
- bad "codex-monitored.sh missing numeric env validator: $helper"
1669
- isolation_missing=1
1670
- fi
1671
- done
1672
-
1673
- if make_temp_dir tmp_env /tmp/codex-monitored-env.XXXXXX; then
1674
- if CODEX_MONITORED_HEARTBEAT=0 CODEX_BIN=/bin/true \
1675
- bash config/skills/_shared/codex-monitored.sh prompt \
1676
- >"$tmp_env/heartbeat.stdout" 2>"$tmp_env/heartbeat.stderr"; then
1677
- bad "codex-monitored.sh accepted CODEX_MONITORED_HEARTBEAT=0"
1678
- isolation_missing=1
1679
- elif ! grep -F 'CODEX_MONITORED_HEARTBEAT must be > 0' "$tmp_env/heartbeat.stderr" >/dev/null 2>&1; then
1680
- bad "codex-monitored.sh heartbeat validation emitted wrong error"
1681
- isolation_missing=1
1682
- fi
1683
- if CODEX_MONITORED_TIMEOUT_SEC=abc CODEX_BIN=/bin/true \
1684
- bash config/skills/_shared/codex-monitored.sh prompt \
1685
- >"$tmp_env/timeout.stdout" 2>"$tmp_env/timeout.stderr"; then
1686
- bad "codex-monitored.sh accepted non-numeric CODEX_MONITORED_TIMEOUT_SEC"
1687
- isolation_missing=1
1688
- elif ! grep -F 'CODEX_MONITORED_TIMEOUT_SEC must be a non-negative integer' "$tmp_env/timeout.stderr" >/dev/null 2>&1; then
1689
- bad "codex-monitored.sh timeout validation emitted wrong error"
1690
- isolation_missing=1
1691
- fi
1692
- rm -rf "$tmp_env"
1693
- else
1694
- isolation_missing=1
1695
- fi
1696
-
1697
- if make_temp_dir tmp_iso /tmp/codex-monitored-isolated.XXXXXX; then
1698
- cat > "$tmp_iso/codex" <<'EOF'
1699
- #!/usr/bin/env bash
1700
- printf '%s\n' "$@" > "$CODEX_FAKE_ARGS_OUT"
1701
- EOF
1702
- chmod +x "$tmp_iso/codex"
1703
- (
1704
- unset DEVLYN_INVOCATION_RUN_ID DEVLYN_INVOCATION_PHASE \
1705
- DEVLYN_INVOCATION_ROUND DEVLYN_INVOCATION_WORKDIR \
1706
- DEVLYN_INVOCATION_PROMPT_FILE DEVLYN_INVOCATION_SESSION_FILE \
1707
- DEVLYN_INVOCATION_RECEIPT
1708
- CODEX_FAKE_ARGS_OUT="$tmp_iso/args.txt" \
1709
- CODEX_MONITORED_ISOLATED=1 \
1710
- CODEX_MONITORED_HEARTBEAT=999 \
1711
- CODEX_BIN="$tmp_iso/codex" \
1712
- bash config/skills/_shared/codex-monitored.sh -s read-only prompt \
1713
- >"$tmp_iso/stdout.txt" 2>"$tmp_iso/stderr.txt"
1714
- )
1715
- iso_exit=$?
1716
- if [ $iso_exit -ne 0 ]; then
1717
- bad "codex-monitored.sh isolated fake invocation exited $iso_exit"
1718
- isolation_missing=1
1719
- else
1720
- for expected in exec --ignore-user-config --ignore-rules --ephemeral --disable codex_hooks --disable hooks -s read-only prompt; do
1721
- if ! grep -Fx -- "$expected" "$tmp_iso/args.txt" >/dev/null 2>&1; then
1722
- bad "codex-monitored.sh isolated fake invocation missing arg: $expected"
1723
- isolation_missing=1
1724
- fi
1725
- done
1726
- if ! grep -F '[codex-monitored] isolated=1' "$tmp_iso/stderr.txt" >/dev/null 2>&1; then
1727
- bad "codex-monitored.sh isolated fake invocation missing lifecycle marker"
1728
- isolation_missing=1
1729
- fi
1730
- fi
1731
- rm -rf "$tmp_iso"
1732
- else
1733
- isolation_missing=1
1734
- fi
1735
- if [ $isolation_missing -eq 0 ]; then
1736
- ok "bounded Codex probe/judge calls require isolated wrapper mode"
1737
- fi
1738
-
1739
- section "Check 10a1: Shared helper paths resolve from the invoked skill"
1740
- shared_path_offenders=$(grep -RInF '.claude/skills/_shared' config/skills 2>/dev/null \
1741
- | grep -v 'roadmap-archival-workspace/' \
1742
- | grep -v 'devlyn:auto-resolve-workspace/' \
1743
- | grep -v 'devlyn:ideate-workspace/' \
1744
- | grep -v 'preflight-workspace/' \
1745
- || true)
1746
- if [ -z "$shared_path_offenders" ]; then
1747
- ok "no project-relative .claude/skills/_shared paths in managed skills"
1748
- else
1749
- while IFS= read -r f; do bad "$f"; done <<< "$shared_path_offenders"
1750
- fi
1751
-
1752
- section "Check 10a2: Bounded non-interactive calls discard inherited input"
1753
- if python3 - <<'PY'
1754
- import json, subprocess, sys
1755
-
1756
- prompt = "sealed argv prompt\n두 번째 줄"
1757
- poison = b"OPERATOR_HEREDOC_SHOULD_NOT_BECOME_USER_INPUT\n"
1758
- child = (
1759
- "import json,sys,time; "
1760
- "print(json.dumps({'argv':sys.argv[1:],'stdin_hex':sys.stdin.buffer.read().hex()},ensure_ascii=False),flush=True); "
1761
- "print('stderr preserved',file=sys.stderr,flush=True); "
1762
- "time.sleep(float(sys.argv[2])); raise SystemExit(int(sys.argv[3]))"
1763
- )
1764
- for label, delay, code, expected_exit in (("normal", 0, 0, 0), ("nonzero", 0, 17, 17), ("timeout", 30, 0, 124)):
1765
- args = [prompt, str(delay), str(code)]
1766
- result = subprocess.run(
1767
- [sys.executable, "config/skills/_shared/run-bounded.py", "1", "--", sys.executable, "-c", child, *args],
1768
- input=poison, capture_output=True, timeout=9,
1769
- )
1770
- assert result.returncode == expected_exit, (label, result.returncode)
1771
- assert json.loads(result.stdout) == {"argv": args, "stdin_hex": ""}, (label, result.stdout)
1772
- assert result.stderr == b"stderr preserved\n", (label, result.stderr)
1773
- PY
1774
- then
1775
- ok "run-bounded.py closes child stdin and preserves argv, streams, exits and timeout"
1776
- else
1777
- bad "run-bounded.py inherited-input/exit/timeout regression failed"
1778
- fi
1779
-
1780
- # ---------------------------------------------------------------------------
1781
- # 10b. Shared routing docs must describe the current 2-skill surface.
1782
- # A stale auto-resolve/preflight/ideate-CHALLENGE reference can misroute
1783
- # bounded pair work back into unisolated or retired Codex paths.
1784
- # ---------------------------------------------------------------------------
1785
- section "Check 10b: Shared routing docs avoid retired skill surfaces"
1786
- offenders=$(grep -RInE 'auto-resolve/SKILL\.md|preflight/SKILL\.md|challenge-rubric\.md|ideate CHALLENGE phase|does NOT consume this file|cross-model challenge phases when configured|phase-1-build\.md|phase-2-evaluate\.md|phase-3-critic\.md' \
1787
- config/skills/_shared 2>/dev/null || true)
1788
- if [ -z "$offenders" ]; then
1789
- ok "shared routing docs reference current devlyn:ideate/devlyn:resolve surface"
1790
- else
1791
- while IFS= read -r f; do bad "$f"; done <<< "$offenders"
1792
- fi
1793
-
1794
- # ---------------------------------------------------------------------------
1795
- # 10c. User-facing current docs must not advertise retired skills.
1796
- # Historical archive mentions are allowed, but install/package copy and
1797
- # active helper scripts should describe ideate -> resolve, not
1798
- # ideate -> auto-resolve -> preflight or ideate CHALLENGE.
1799
- # ---------------------------------------------------------------------------
1800
- section "Check 10c: User-facing current docs avoid retired skill surfaces"
1801
- offenders=$(
1802
- {
1803
- sed '/<!-- legacy-surface-map:begin/,/<!-- legacy-surface-map:end/s/.*//' README.md 2>/dev/null \
1804
- | grep -nE '/devlyn:auto-resolve|ideate CHALLENGE|--with-codex|Quick Start pointing to ideate → auto-resolve → preflight|auto-resolve'\''s build agent|Core pipeline skills \(`ideate`, `auto-resolve`, `preflight`\)' || true
1805
- grep -nE '"description": .*auto-resolve|"description": .*preflight' package.json 2>/dev/null || true
1806
- grep -nE 'so auto-resolve doesn'\''t prompt' bin/devlyn.js 2>/dev/null || true
1807
- grep -nE 'devlyn:auto-resolve|phase-1-build\.md|phase-2-evaluate\.md|phase-3-critic\.md' scripts/static-ab.sh 2>/dev/null || true
1808
- } | sed -E 's#^#user-facing retired-surface reference: #'
1809
- )
1810
- if [ -z "$offenders" ]; then
1811
- ok "README/package/helper copy describes current ideate -> resolve surface"
1812
- else
1813
- while IFS= read -r f; do bad "$f"; done <<< "$offenders"
1814
- fi
1815
-
1816
- # ---------------------------------------------------------------------------
1817
- # 10d. Prompt adapters cite the current official model guidance.
1818
- # Prompt edits must be model-specific where tactics differ: OpenAI guidance
1819
- # for GPT/Codex, Anthropic guidance for Claude. This check keeps that
1820
- # contract from becoming oral tradition.
1821
- # ---------------------------------------------------------------------------
1822
- section "Check 10d: Prompt adapters cite official model guidance"
1823
- adapter_missing=0
1824
- if ! grep -Fq 'https://developers.openai.com/api/docs/guides/prompt-guidance?model=gpt-5.5' \
1825
- config/skills/_shared/adapters/codex.md; then
1826
- bad "codex adapter missing official OpenAI prompt guidance URL"
1827
- adapter_missing=1
1828
- fi
1829
- if ! grep -Fq 'https://platform.claude.com/docs/en/build-with-claude/prompt-engineering/claude-prompting-best-practices' \
1830
- config/skills/_shared/adapters/claude.md; then
1831
- bad "claude adapter missing official Claude prompting best-practices URL"
1832
- adapter_missing=1
1833
- fi
1834
- for pattern in 'Use Markdown only where it carries structure' 'metaprompter for itself'; do
1835
- if ! grep -Fq "$pattern" config/skills/_shared/adapters/codex.md; then
1836
- bad "codex adapter missing official-guidance tactic: $pattern"
1837
- adapter_missing=1
1838
- fi
1839
- done
1840
- for pattern in 'high` or `xhigh` effort' 'report every issue you find' 'do not filter for importance or confidence' 'prefer concise positive examples' '<example>'; do
1841
- if ! grep -Fq "$pattern" config/skills/_shared/adapters/claude.md; then
1842
- bad "claude adapter missing official-guidance tactic: $pattern"
1843
- adapter_missing=1
1844
- fi
1845
- done
1846
- for file in config/skills/devlyn:resolve/SKILL.md config/skills/devlyn:ideate/SKILL.md; do
1847
- if ! grep -Fq '_shared/adapters/<engine>.md' "$file"; then
1848
- bad "$file — missing per-engine adapter injection contract"
1849
- adapter_missing=1
1850
- fi
1851
- done
1852
- if [ $adapter_missing -eq 0 ]; then
1853
- ok "adapters cite official GPT/Claude guidance, carry model-specific tactics, and both skills inject them"
1854
- fi
1855
-
1856
- # ---------------------------------------------------------------------------
1857
- # 10d1. Opus sidecar must fail closed on score-source mapping before provider calls.
1858
- # ---------------------------------------------------------------------------
1859
- section "Check 10d1: Opus judge sidecar validates blind mapping before provider calls"
1860
- if ! grep -Fq 'judge blind mapping missing arm(s)' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
1861
- || ! grep -Fq 'scores_by_arm without blind mapping' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
1862
- || ! grep -Fq 'scores_by_arm malformed score(s)' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
1863
- || ! grep -Fq 'gpt judge.json _blind_mapping must be an object' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
1864
- || ! grep -Fq 'def is_score(value):' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
1865
- || ! grep -Fq 'not isinstance(value, bool) and 0 <= value <= 100' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
1866
- || ! grep -Fq 'invalid opus score value(s)' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
1867
- || ! grep -Fq 'invalid opus disqualifier value(s)' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
1868
- || ! grep -Fq 'opus-invalid-generated-dq' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
1869
- || ! grep -Fq 'def blind_mapping(j):' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
1870
- || ! grep -Fq 'def mapped_arm_set(j):' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
1871
- || ! grep -Fq '{"solo_claude", "bare"}.issubset(mapped_arms)' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
1872
- || ! grep -Fq 'def mapped_scores(j):' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
1873
- || ! grep -Fq 'if arm in mapped_arms and is_score(score)' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
1874
- || ! grep -Fq 'def margin_from_scores(scores, left, right):' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
1875
- || ! grep -Fq 'def mapped_winner(j, scores):' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
1876
- || ! grep -Fq 'def fmt_metric(value):' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
1877
- || ! grep -Fq 'g_l1_l0 = margin_from_scores(g_scores, "solo_claude", "bare")' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
1878
- || ! grep -Fq 'g_v_l0 = margin_from_scores(g_scores, "variant", "bare")' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
1879
- || ! grep -Fq 'g_winner = mapped_winner(g, g_scores)' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
1880
- || ! grep -Fq '"winner_agree": g_winner is not None and o_winner is not None and g_winner == o_winner' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
1881
- || ! grep -Fq '"gpt_scores": g_scores' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
1882
- || ! grep -Fq '"opus_scores": o_scores' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
1883
- || ! grep -Fq 'opus-bad-mapping' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
1884
- || ! grep -Fq 'opus-malformed-mapping' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
1885
- || ! grep -Fq 'opus-malformed-score' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
1886
- || ! grep -Fq 'opus-invalid-generated-score' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
1887
- || ! grep -Fq 'arg-parse-opus-summary-mapping' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
1888
- || ! grep -Fq 'arg-parse-opus-summary-null-margin' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
1889
- || ! grep -Fq 'assert row["gpt_margin_l1_l0"] == 10' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
1890
- || ! grep -Fq 'assert row["gpt_winner"] is None' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
1891
- || ! grep -Fq "gpt_l1_l0_avg=na" benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
1892
- || ! grep -Fq 'solo_claude-bare={chosen' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
1893
- || ! grep -Fq 'variant-solo_claude={chosen' benchmark/auto-resolve/scripts/judge-opus-pass.sh; then
1894
- offenders="${offenders}"$'\n'"judge-opus-pass.sh must validate _blind_mapping locally before invoking Claude"
1895
- fi
1896
-
1897
- # ---------------------------------------------------------------------------
1898
- # 10e. Benchmark docs must describe the current solo-vs-pair topology.
1899
- # Pair evidence work depends on the bare/solo_claude/pair contract being current:
1900
- # bare, solo_claude, and the selected pair arm are measured today.
1901
- # ---------------------------------------------------------------------------
1902
- section "Check 10e: Benchmark docs describe current 3-arm pair topology"
1903
- offenders=$(grep -RInE 'L1 .*queued|cannot directly verify the L1|auto-resolve → preflight|ideate → auto-resolve|all fixtures × 2 arms|9 fixtures × 2 arms|9 fixtures × 3 arms|≥ 7 of 9|7/9 fixtures|variant` / `bare|variant/\{input|preflight; bare|Audited by preflight|Variant'\''s CRITIC|future enhancement|release-blocker|Today the suite runs `variant`|/devlyn:auto-resolve|auto-resolve-ready|REAL auto-resolve|real auto-resolve|auto-resolve run|auto-resolve runs|Claude/auto-resolve|variant − bare|Margin \(variant|Ship thresholds use margin \(variant|both arms improve together|both arms|benchmark --n 3|run-suite\.sh --n 3|higher confidence for ship decisions|3 runs per fixture for ship decisions|One-command A/B benchmark|A/B randomized|A/B benchmark suite vs bare|not vs bare — bare is the opponent' \
1904
- benchmark/auto-resolve/BENCHMARK-DESIGN.md \
1905
- benchmark/auto-resolve/README.md \
1906
- benchmark/auto-resolve/RUBRIC.md \
1907
- benchmark/auto-resolve/BENCHMARK-RESULTS.md \
1908
- benchmark/auto-resolve/run-real-benchmark.md \
1909
- benchmark/auto-resolve/fixtures/SCHEMA.md \
1910
- benchmark/auto-resolve/shadow-fixtures/README.md \
1911
- 2>/dev/null || true)
1912
- fixture_note_stale=$(git grep -InE -- 'both arms|both solo and pair arms|bare or solo consistently reaches ceiling' -- \
1913
- 'benchmark/auto-resolve/fixtures/**/NOTES.md' \
1914
- ':!benchmark/auto-resolve/fixtures/retired/**' \
1915
- 2>/dev/null || true)
1916
- if [ -n "$fixture_note_stale" ]; then
1917
- offenders="${offenders}"$'\n'"fixture NOTES must name bare and solo_claude explicitly instead of ambiguous arm wording:"$'\n'"$fixture_note_stale"
1918
- fi
1919
- shadow_note_stale=$(git grep -In -- 'solo headroom' -- \
1920
- 'benchmark/auto-resolve/shadow-fixtures/**/NOTES.md' \
1921
- 2>/dev/null || true)
1922
- if [ -n "$shadow_note_stale" ]; then
1923
- offenders="${offenders}"$'\n'"shadow fixture NOTES must name solo_claude headroom explicitly:"$'\n'"$shadow_note_stale"
1924
- fi
1925
- active_doc_stale_solo_scores=$(git grep -InE -- '(^|[^[:alnum:]_])solo [0-9]+|/ solo$|, solo [0-9]+|vs solo [0-9]+' -- \
1926
- benchmark/auto-resolve/BENCHMARK-RESULTS.md \
1927
- benchmark/auto-resolve/README.md \
1928
- benchmark/auto-resolve/run-real-benchmark.md \
1929
- 2>/dev/null || true)
1930
- if [ -n "$active_doc_stale_solo_scores" ]; then
1931
- offenders="${offenders}"$'\n'"active benchmark score evidence must name solo_claude instead of shorthand solo:"$'\n'"$active_doc_stale_solo_scores"
1932
- fi
1933
- active_stale_margin_labels=$(git grep -InE -- 'l1-l0=|v-l1=|L1-L0 disagreement|Per-axis L1-L0|Suite-level per-axis L1-L0|Suite avg L1-L0|L2-L1 margin' -- \
1934
- benchmark/auto-resolve/BENCHMARK-DESIGN.md \
1935
- benchmark/auto-resolve/README.md \
1936
- benchmark/auto-resolve/RUBRIC.md \
1937
- benchmark/auto-resolve/scripts/judge-opus-pass.sh \
1938
- 2>/dev/null || true)
1939
- if [ -n "$active_stale_margin_labels" ]; then
1940
- offenders="${offenders}"$'\n'"active benchmark docs/stdout must use key-aligned margin labels:"$'\n'"$active_stale_margin_labels"
1941
- fi
1942
- if grep -RInE 'gate: ≥ 7 of 9|Hard floor 3: ≥ 7 of 9|7-of-9 L1 floor|all 9 fixtures produced' \
1943
- benchmark/auto-resolve/scripts/compile-report.py \
1944
- benchmark/auto-resolve/scripts/ship-gate.py \
1945
- benchmark/auto-resolve/scripts/judge-opus-pass.sh >/dev/null 2>&1; then
1946
- offenders="${offenders}"$'\n'"benchmark scripts must describe the current extended fixture count as an explicit selected/gated set, not stale 9-fixture wording"
1947
- fi
1948
- if grep -Fq 'PAIR_ARM="l2_gated"' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh; then
1949
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh: default pair arm must stay on current measured l2_risk_probes path"
1950
- fi
1951
- if ! grep -Fq 'for required in result.json verify.json diff.patch; do' \
1952
- benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
1953
- || ! grep -Fq 'reuse source missing $required' \
1954
- benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh; then
1955
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh: calibrated-arm reuse must fail closed on missing result.json, verify.json, and diff.patch"
1956
- fi
1957
- if ! grep -Fq 'reuse destination incomplete $required' \
1958
- benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
1959
- || ! grep -Fq 'reuse destination is not a directory' \
1960
- benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh; then
1961
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh: calibrated-arm reuse must fail closed on incomplete existing destination"
1962
- fi
1963
- if ! grep -Fq 'diff.patch missing' benchmark/auto-resolve/scripts/headroom-gate.py \
1964
- || ! grep -Fq 'diff.patch missing' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py; then
1965
- offenders="${offenders}"$'\n'"benchmark pair gates must require diff.patch artifacts for measured arms"
1966
- fi
1967
- if ! grep -Fq 'pair-arm must be l2_risk_probes or l2_gated' \
1968
- benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
1969
- || ! grep -Fq 'pair-arm l2_forced is retired' \
1970
- benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh; then
1971
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh: pair arm selection must fail closed before fixture execution"
1972
- fi
1973
- if ! grep -Fq 'l2_risk_probes|l2_gated) ;;' \
1974
- benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh; then
1975
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh: runner allowlist must remain l2_risk_probes|l2_gated only"
1976
- fi
1977
- if grep -Fq 'default="l2_gated"' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py; then
1978
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py: --pair-arm default must stay on current measured l2_risk_probes path"
1979
- fi
1980
- if ! grep -Fq 'from pair_evidence_contract import (' \
1981
- benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
1982
- || ! grep -Fq 'ALLOWED_PAIR_ARMS,' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
1983
- || ! grep -Fq 'ALLOWED_PAIR_ARMS = {"l2_risk_probes", "l2_gated"}' \
1984
- benchmark/auto-resolve/scripts/pair_evidence_contract.py \
1985
- || ! grep -Fq 'pair-arm l2_forced is retired' \
1986
- benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py; then
1987
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py: pair arm selection must fail closed inside the gate"
1988
- fi
1989
- if ! grep -Fq '"l2_risk_probes"' benchmark/auto-resolve/scripts/check-f9-artifacts.py; then
1990
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/check-f9-artifacts.py: F9 skill-driven artifact check must accept current l2_risk_probes arm"
1991
- fi
1992
- if ! grep -Fq '_load_json_object' benchmark/auto-resolve/scripts/check-f9-artifacts.py \
1993
- || ! grep -Fq 'expected JSON object' benchmark/auto-resolve/scripts/test-check-f9-artifacts.sh; then
1994
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/check-f9-artifacts.py: F9 timing/state JSON must fail closed on non-object payloads"
1995
- fi
1996
- if grep -RInE 'asserts variant/solo|Variant-only artifact checks|Variant artifact check' \
1997
- benchmark/auto-resolve/scripts/check-f9-artifacts.py \
1998
- benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/NOTES.md \
1999
- benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/spec.md >/dev/null 2>&1; then
2000
- offenders="${offenders}"$'\n'"F9 artifact docs/checker wording must describe skill-driven arms, not variant-only checks"
2001
- fi
2002
- if grep -Fq '<variant|bare>' benchmark/auto-resolve/scripts/run-fixture.sh; then
2003
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/run-fixture.sh: usage must list current benchmark arms"
2004
- fi
2005
- if grep -Fq 'l2_gated/l2_forced' benchmark/auto-resolve/scripts/run-fixture.sh; then
2006
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/run-fixture.sh: comments must not omit l2_risk_probes from l2_* arm handling"
2007
- fi
2008
- if grep -Fq 'ENGINE_CLAUSE="--engine auto"' benchmark/auto-resolve/scripts/run-fixture.sh \
2009
- || grep -Fq 'Run with `--engine auto`' benchmark/auto-resolve/scripts/run-fixture.sh; then
2010
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/run-fixture.sh: smoke variant arm must use current --engine claude --risk-probes path, not retired --engine auto"
2011
- fi
2012
- if ! grep -Fq 'ENGINE_CLAUSE="--engine claude --risk-probes"' benchmark/auto-resolve/scripts/run-fixture.sh; then
2013
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/run-fixture.sh: variant/l2 risk-probes path must remain available"
2014
- fi
2015
- if ! grep -Fq '{bare, solo_claude, selected pair arm}' benchmark/auto-resolve/scripts/judge.sh; then
2016
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/judge.sh: blind judge topology comment must describe current pair-candidate proof runs"
2017
- fi
2018
- if grep -Fq 'blind judge scores l2_gated' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
2019
- || grep -Fq 'l2_gated is clean' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py; then
2020
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py: gate docstring must describe selected pair arm, not l2_gated-only proof"
2021
- fi
2022
- if grep -Fq 'only then spends a `l2_gated` arm' benchmark/auto-resolve/README.md \
2023
- || grep -Fq 'fresh `l2_gated` measurement' benchmark/auto-resolve/README.md; then
2024
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/README.md: full-pipeline runner docs must not present l2_gated as the default measured proof path"
2025
- fi
2026
- if ! grep -Fq 'Pair arms are limited to current' benchmark/auto-resolve/README.md \
2027
- || ! grep -Fq '`l2_forced` is' benchmark/auto-resolve/README.md; then
2028
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/README.md: full-pipeline docs must state allowed pair arms and retired l2_forced"
2029
- fi
2030
- if ! grep -Fq 'Current solo<pair' benchmark/auto-resolve/RUBRIC.md \
2031
- || ! grep -Fq 'evidence uses the full-pipeline pair gate' benchmark/auto-resolve/RUBRIC.md \
2032
- || ! grep -Fq 'explicit selected pair arm' benchmark/auto-resolve/RUBRIC.md \
2033
- || ! grep -Fq 'selected pair arm over `solo_claude`' benchmark/auto-resolve/RUBRIC.md; then
2034
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/RUBRIC.md: rubric must distinguish legacy run-suite ship gate from current selected-pair-arm evidence"
2035
- fi
2036
- if grep -Fq 'PHASE 8' benchmark/auto-resolve/run-real-benchmark.md \
2037
- || grep -Fq 'security_review on `--engine auto`' benchmark/auto-resolve/run-real-benchmark.md \
2038
- || grep -Fq 'BENCHMARK-RESULTS-v3.md' benchmark/auto-resolve/run-real-benchmark.md; then
2039
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/run-real-benchmark.md: real-run docs must describe the current 5-phase resolve and pair score harness"
2040
- fi
2041
- if ! grep -Fq 'Archive note (2026-05-14): historical pre-cutover benchmark plan' benchmark/auto-resolve/v3.6-ab-plan.md \
2042
- || ! grep -Fq 'Do not use this file for current' benchmark/auto-resolve/v3.6-ab-plan.md \
2043
- || ! grep -Fq 'Archive note (2026-05-14): historical pre-cutover results' benchmark/auto-resolve/BENCHMARK-RESULTS-v3.md \
2044
- || ! grep -Fq 'Do not treat these projected' benchmark/auto-resolve/BENCHMARK-RESULTS-v3.md \
2045
- || ! grep -Fq 'Archive note (2026-05-14): historical v3.2 pilot' benchmark/auto-resolve/PILOT-RESULTS-v3.2.md \
2046
- || ! grep -Fq 'This n=1 pilot is not current solo<pair evidence' benchmark/auto-resolve/PILOT-RESULTS-v3.2.md \
2047
- || ! grep -Fq 'Archive note (2026-05-14): historical v3.2 strict-route pilot' benchmark/auto-resolve/PILOT-RESULTS-STRICT-v3.2.md \
2048
- || ! grep -Fq 'This inline n=1 pilot is not current solo<pair evidence' benchmark/auto-resolve/PILOT-RESULTS-STRICT-v3.2.md \
2049
- || ! grep -Fq 'archived static comparison helper for pre-cutover auto-resolve' benchmark/auto-resolve/measure-static.py \
2050
- || ! grep -Fq 'is not current solo<pair evidence' benchmark/auto-resolve/measure-static.py; then
2051
- offenders="${offenders}"$'\n'"archived v3 benchmark artifacts must be clearly marked as non-current solo<pair evidence"
2052
- fi
2053
- if ! grep -Fq 'l2_risk_probes` | current measured pair path' benchmark/auto-resolve/run-real-benchmark.md \
2054
- || ! grep -Fq 'Dry-runs, lint, and shell tests prove wiring only' benchmark/auto-resolve/run-real-benchmark.md; then
2055
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/run-real-benchmark.md: real-run docs must name the measured pair arm and distinguish wiring checks from scores"
2056
- fi
2057
- if grep -RInE 'F27.*unmeasured|unmeasured.*F27' \
2058
- benchmark/auto-resolve/README.md \
2059
- benchmark/auto-resolve/BENCHMARK-RESULTS.md \
2060
- benchmark/auto-resolve/run-real-benchmark.md \
2061
- benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/NOTES.md >/dev/null 2>&1; then
2062
- offenders="${offenders}"$'\n'"F27 docs must record the measured headroom failure, not stale unmeasured-candidate wording"
2063
- fi
2064
- if [ -d benchmark/auto-resolve/fixtures/F27-cli-subscription-proration ]; then
2065
- offenders="${offenders}"$'\n'"F27 must stay out of active golden fixtures after failing headroom; keep it under fixtures/retired/"
2066
- fi
2067
- if [ ! -f benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/RETIRED.md ]; then
2068
- offenders="${offenders}"$'\n'"retired F27 must keep RETIRED.md with the measured rejection reason"
2069
- fi
2070
- if ! grep -Fq '"benchmark/auto-resolve/fixtures/retired/F*/**"' package.json; then
2071
- offenders="${offenders}"$'\n'"package.json must include retired fixtures so replay artifacts ship in npm packages"
2072
- fi
2073
- if ! grep -Fq '20260511-f27-headroom-smoke-061401' benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/NOTES.md \
2074
- || ! grep -Fq '20260511-f27-headroom-smoke-061401' benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/RETIRED.md; then
2075
- offenders="${offenders}"$'\n'"F27 notes must cite the measured headroom smoke run"
2076
- fi
2077
- if grep -Fq 'better candidate for pair risk probes' benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/NOTES.md; then
2078
- offenders="${offenders}"$'\n'"F27 notes must not describe rejected F27 as a better pair-risk-probe candidate"
2079
- fi
2080
- if grep -Fq 'F16-cli-quote-tax-rules F27-cli-subscription-proration' \
2081
- benchmark/auto-resolve/README.md benchmark/auto-resolve/run-real-benchmark.md; then
2082
- offenders="${offenders}"$'\n'"benchmark docs must not recommend rejected F27 in headroom/full-pipeline command examples"
2083
- fi
2084
- if [ -d benchmark/auto-resolve/fixtures/F28-cli-return-authorization ]; then
2085
- offenders="${offenders}"$'\n'"F28 must stay out of active golden fixtures after corrected-oracle reverify failed headroom; keep it under fixtures/retired/"
2086
- fi
2087
- if [ ! -f benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/RETIRED.md ]; then
2088
- offenders="${offenders}"$'\n'"retired F28 must keep RETIRED.md with the corrected-oracle rejection reason"
2089
- fi
2090
- if ! grep -Fq '20260511-f28-headroom-smoke-085307' benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/NOTES.md \
2091
- || ! grep -Fq '20260511-f28-pair-smoke-091021' benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/NOTES.md \
2092
- || ! grep -Fq '20260511-f28-policy-oraclefix-reverified-pair' benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/NOTES.md \
2093
- || ! grep -Fq '20260511-f28-policy-oraclefix-reverified-pair' benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/RETIRED.md \
2094
- || ! grep -Fq '20260511-f28-headroom-smoke-085307' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
2095
- || ! grep -Fq '20260511-f28-pair-smoke-091021' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
2096
- || ! grep -Fq '20260511-f28-headroom-smoke-085307' benchmark/auto-resolve/README.md \
2097
- || ! grep -Fq '20260511-f28-pair-smoke-091021' benchmark/auto-resolve/README.md \
2098
- || ! grep -Fq '20260511-f28-headroom-smoke-085307' benchmark/auto-resolve/run-real-benchmark.md \
2099
- || ! grep -Fq '20260511-f28-pair-smoke-091021' benchmark/auto-resolve/run-real-benchmark.md; then
2100
- offenders="${offenders}"$'\n'"F28 docs must cite measured smoke and corrected-oracle rejection run ids before anyone counts it"
2101
- fi
2102
- if grep -Fq 'F16-cli-quote-tax-rules F28-cli-return-authorization' \
2103
- benchmark/auto-resolve/README.md benchmark/auto-resolve/run-real-benchmark.md README.md; then
2104
- offenders="${offenders}"$'\n'"benchmark docs must not recommend unstable F28 in pair-evidence command examples"
2105
- fi
2106
- if ! bash -n benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh >/dev/null 2>&1; then
2107
- offenders="${offenders}"$'\n'"pair-rejected-fixtures.sh must be valid bash"
2108
- fi
2109
- if ! grep -Fq 'trivial calibration fixture' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
2110
- || ! grep -Fq '20260512-f2-medium-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
2111
- || ! grep -Fq '20260511-f3-http-error-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
2112
- || ! grep -Fq '20260512-f4-web-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
2113
- || ! grep -Fq '20260512-f5-fixloop-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
2114
- || ! grep -Fq '20260512-f6-checksum-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
2115
- || ! grep -Fq '20260512-f7-scope-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
2116
- || ! grep -Fq 'known-limit ambiguity fixture' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
2117
- || ! grep -Fq '20260512-f9-e2e-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
2118
- || ! grep -Fq '20260507-f10-f11-tier1-full-pipeline' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
2119
- || ! grep -Fq '20260511-f12-webhook-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
2120
- || ! grep -Fq '20260511-f15-concurrency-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
2121
- || ! grep -Fq 'bare 94 / solo_claude 98 in 20260508-f22-exact-error-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
2122
- || ! grep -Fq '20260508-f26-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
2123
- || ! grep -Fq '20260511-f27-headroom-smoke-061401' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
2124
- || ! grep -Fq '20260511-f28-policy-oraclefix-reverified-pair' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
2125
- || ! grep -Fq '20260510-f29-headroom-v2' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
2126
- || ! grep -Fq '20260511-f30-headroom-v1' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
2127
- || ! grep -Fq '20260512-f31-seat-rebalance-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
2128
- || ! grep -Fq '20260512-f32-subscription-renewal-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
2129
- || ! grep -Fq '20260513-s2-inventory-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
2130
- || ! grep -Fq '20260513-s3-ticket-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
2131
- || ! grep -Fq '20260513-s4-return-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
2132
- || ! grep -Fq '20260513-s5-credit-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
2133
- || ! grep -Fq '20260514-s6-refund-headroom-v1' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
2134
- || ! grep -Fq 'source "$BENCH_ROOT/scripts/pair-rejected-fixtures.sh"' benchmark/auto-resolve/scripts/run-headroom-candidate.sh \
2135
- || ! grep -Fq 'source "$BENCH_ROOT/scripts/pair-rejected-fixtures.sh"' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
2136
- || ! grep -Fq 'fixture_smoke_only' benchmark/auto-resolve/scripts/run-headroom-candidate.sh \
2137
- || ! grep -Fq 'fixture_smoke_only' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
2138
- || ! grep -Fq 'smoke-only-s1-provider-run' benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh \
2139
- || ! grep -Fq 'smoke-only-s1-provider-run' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
2140
- || ! grep -Fq 'smoke-only-s1-cli-headroom' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
2141
- || ! grep -Fq 'smoke-only-s1-cli-pair' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
2142
- || ! grep -Fq 'declare -F rejected_pair_fixture_reason' benchmark/auto-resolve/scripts/run-headroom-candidate.sh \
2143
- || ! grep -Fq 'declare -F rejected_pair_fixture_reason' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
2144
- || ! grep -Fq 'rejected-f31-fixture' benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh \
2145
- || ! grep -Fq 'rejected-f32-fixture' benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh \
2146
- || ! grep -Fq 'rejected-s6-shadow-fixture' benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh \
2147
- || ! grep -Fq 'rejected-f31-fixture' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
2148
- || ! grep -Fq 'rejected-f32-fixture' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
2149
- || ! grep -Fq 'rejected-s6-shadow-fixture' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
2150
- || ! grep -Fq 'load_rejected_short_ids' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
2151
- || ! grep -Fq 'load_rejected_fixture_reasons' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
2152
- || ! grep -Fq 'rejected_excluded' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
2153
- || ! grep -Fq 'rejected_excluded_reasons' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
2154
- || ! grep -Fq 'rejected_excluded_reasons' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
2155
- || ! grep -Fq 'selection_rule.rejected_excluded_reasons' benchmark/auto-resolve/README.md \
2156
- || ! grep -Fq 'rejected_excluded_reasons' benchmark/auto-resolve/run-real-benchmark.md \
2157
- || ! grep -Fq 'fixtures_pair_eligible must not be empty' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
2158
- || ! grep -Fq 'gate3_threshold_count must be a positive integer' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
2159
- || ! grep -Fq 'selection_rule.rejected_excluded_reasons keys must match rejected_excluded' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
2160
- || ! grep -Fq 'nan-threshold-manifest' benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh \
2161
- || ! grep -Fq 'bad-rejected-reasons-manifest' benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh \
2162
- || ! grep -Fq 'fixture rejected for pair-candidate runs' benchmark/auto-resolve/scripts/headroom-gate.py \
2163
- || ! grep -Fq 'fixture rejected for pair-candidate runs' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
2164
- || ! grep -Fq 'rejected fixture registry missing' benchmark/auto-resolve/scripts/headroom-gate.py \
2165
- || ! grep -Fq 'rejected fixture registry missing' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
2166
- || ! grep -Fq '([FS]\d+)-\*\|([FS]\d+)' benchmark/auto-resolve/scripts/headroom-gate.py \
2167
- || ! grep -Fq '([FS]\d+)-\*\|([FS]\d+)' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
2168
- || ! grep -Fq '([FS]\d+)-\*\|([FS]\d+)' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
2169
- || ! grep -Fq '([FS]\d+)-\*\|([FS]\d+)' benchmark/auto-resolve/scripts/audit-headroom-rejections.py \
2170
- || ! grep -Fq '([FS]\d+)-\*\|([FS]\d+)' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
2171
- || ! grep -Fq 'missing-rejected-registry' benchmark/auto-resolve/scripts/test-headroom-gate.sh \
2172
- || ! grep -Fq 'missing-rejected-registry' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
2173
- || ! grep -Fq 'rejected-shadow-direct' benchmark/auto-resolve/scripts/test-headroom-gate.sh \
2174
- || ! grep -Fq 'rejected-shadow-direct' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
2175
- || ! grep -Fq 's-only-registry' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
2176
- || ! grep -Fq 's-only-registry' benchmark/auto-resolve/scripts/test-audit-headroom-rejections.sh \
2177
- || ! grep -Fq 'shadow solo ceiling' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
2178
- || grep -Fq 'case "$fid" in' benchmark/auto-resolve/scripts/run-headroom-candidate.sh \
2179
- || grep -Fq 'case "$fid" in' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh; then
2180
- offenders="${offenders}"$'\n'"pair candidate runners, audits, and manifest builder must honor the shared rejected fixture registry, including F* fixtures and S* shadow controls, without duplicating the case table"
2181
- fi
2182
- if ! grep -Fq 'Pair-candidate status: rejected by design' benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/NOTES.md \
2183
- || ! grep -Fq 'Pair-candidate status: rejected by design' benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/NOTES.md \
2184
- || ! grep -Fq 'rejected by design' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
2185
- || ! grep -Fq 'calibration/known-limit controls' benchmark/auto-resolve/README.md \
2186
- || ! grep -Fq 'calibration/known-limit controls' benchmark/auto-resolve/run-real-benchmark.md; then
2187
- offenders="${offenders}"$'\n'"F1/F8 docs must mark calibration and known-limit controls as rejected by design for pair evidence"
2188
- fi
2189
- if ! grep -Fq '20260512-f2-medium-headroom' benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/NOTES.md \
2190
- || ! grep -Fq '20260512-f2-medium-headroom' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
2191
- || ! grep -Fq '20260512-f2-medium-headroom' benchmark/auto-resolve/README.md \
2192
- || ! grep -Fq '20260512-f2-medium-headroom' benchmark/auto-resolve/run-real-benchmark.md; then
2193
- offenders="${offenders}"$'\n'"F2 docs must cite the measured headroom rejection before anyone counts it"
2194
- fi
2195
- if ! grep -Fq '20260511-f12-webhook-headroom' benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/NOTES.md \
2196
- || ! grep -Fq '20260511-f12-webhook-headroom' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
2197
- || ! grep -Fq '20260511-f12-webhook-headroom' benchmark/auto-resolve/README.md \
2198
- || ! grep -Fq '20260511-f12-webhook-headroom' benchmark/auto-resolve/run-real-benchmark.md; then
2199
- offenders="${offenders}"$'\n'"F12 docs must cite the measured headroom rejection before anyone counts it"
2200
- fi
2201
- if ! grep -Fq '20260507-f10-f11-tier1-full-pipeline' benchmark/auto-resolve/fixtures/F10-persist-write-collision/NOTES.md \
2202
- || ! grep -Fq '20260507-f10-f11-tier1-full-pipeline' benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/NOTES.md \
2203
- || ! grep -Fq '20260507-f10-f11-tier1-full-pipeline' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
2204
- || ! grep -Fq '20260507-f10-f11-tier1-full-pipeline' benchmark/auto-resolve/README.md \
2205
- || ! grep -Fq '20260507-f10-f11-tier1-full-pipeline' benchmark/auto-resolve/run-real-benchmark.md; then
2206
- offenders="${offenders}"$'\n'"F10/F11 docs must cite the measured headroom rejection before anyone counts them"
2207
- fi
2208
- if ! grep -Fq '20260511-f3-http-error-headroom' benchmark/auto-resolve/fixtures/F3-backend-contract-risk/NOTES.md \
2209
- || ! grep -Fq '20260511-f3-http-error-headroom' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
2210
- || ! grep -Fq '20260511-f3-http-error-headroom' benchmark/auto-resolve/README.md \
2211
- || ! grep -Fq '20260511-f3-http-error-headroom' benchmark/auto-resolve/run-real-benchmark.md; then
2212
- offenders="${offenders}"$'\n'"F3 docs must cite the measured headroom rejection before anyone counts it"
2213
- fi
2214
- if ! grep -Fq '20260512-f4-web-headroom' benchmark/auto-resolve/fixtures/F4-web-browser-design/NOTES.md \
2215
- || ! grep -Fq '20260512-f4-web-headroom' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
2216
- || ! grep -Fq '20260512-f4-web-headroom' benchmark/auto-resolve/README.md \
2217
- || ! grep -Fq '20260512-f4-web-headroom' benchmark/auto-resolve/run-real-benchmark.md; then
2218
- offenders="${offenders}"$'\n'"F4 docs must cite the measured headroom rejection before anyone counts it"
2219
- fi
2220
- if ! grep -Fq '20260512-f5-fixloop-headroom' benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/NOTES.md \
2221
- || ! grep -Fq '20260512-f5-fixloop-headroom' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
2222
- || ! grep -Fq '20260512-f5-fixloop-headroom' benchmark/auto-resolve/README.md \
2223
- || ! grep -Fq '20260512-f5-fixloop-headroom' benchmark/auto-resolve/run-real-benchmark.md; then
2224
- offenders="${offenders}"$'\n'"F5 docs must cite the measured headroom rejection before anyone counts it"
2225
- fi
2226
- if ! grep -Fq '20260512-f6-checksum-headroom' benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/NOTES.md \
2227
- || ! grep -Fq '20260512-f6-checksum-headroom' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
2228
- || ! grep -Fq '20260512-f6-checksum-headroom' benchmark/auto-resolve/README.md \
2229
- || ! grep -Fq '20260512-f6-checksum-headroom' benchmark/auto-resolve/run-real-benchmark.md; then
2230
- offenders="${offenders}"$'\n'"F6 docs must cite the measured headroom rejection before anyone counts it"
2231
- fi
2232
- if ! grep -Fq '20260512-f7-scope-headroom' benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/NOTES.md \
2233
- || ! grep -Fq '20260512-f7-scope-headroom' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
2234
- || ! grep -Fq '20260512-f7-scope-headroom' benchmark/auto-resolve/README.md \
2235
- || ! grep -Fq '20260512-f7-scope-headroom' benchmark/auto-resolve/run-real-benchmark.md; then
2236
- offenders="${offenders}"$'\n'"F7 docs must cite the measured headroom rejection before anyone counts it"
2237
- fi
2238
- if ! grep -Fq '20260512-f9-e2e-headroom' benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/NOTES.md \
2239
- || ! grep -Fq '20260512-f9-e2e-headroom' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
2240
- || ! grep -Fq '20260512-f9-e2e-headroom' benchmark/auto-resolve/README.md \
2241
- || ! grep -Fq '20260512-f9-e2e-headroom' benchmark/auto-resolve/run-real-benchmark.md; then
2242
- offenders="${offenders}"$'\n'"F9 docs must cite the measured headroom rejection before anyone counts it"
2243
- fi
2244
- if ! grep -Fq '20260511-f15-concurrency-headroom' benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/NOTES.md \
2245
- || ! grep -Fq '20260511-f15-concurrency-headroom' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
2246
- || ! grep -Fq '20260511-f15-concurrency-headroom' benchmark/auto-resolve/README.md \
2247
- || ! grep -Fq '20260511-f15-concurrency-headroom' benchmark/auto-resolve/run-real-benchmark.md; then
2248
- offenders="${offenders}"$'\n'"F15 docs must cite the measured headroom rejection before anyone counts it"
2249
- fi
2250
- if ! grep -Fq '20260512-f31-seat-rebalance-headroom' benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/NOTES.md \
2251
- || ! grep -Fq '20260512-f31-seat-rebalance-headroom' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
2252
- || ! grep -Fq '20260512-f31-seat-rebalance-headroom' benchmark/auto-resolve/README.md \
2253
- || ! grep -Fq '20260512-f31-seat-rebalance-headroom' benchmark/auto-resolve/run-real-benchmark.md; then
2254
- offenders="${offenders}"$'\n'"F31 docs must cite the measured headroom rejection before anyone counts it"
2255
- fi
2256
- if grep -Fq 'execFileSync' benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/verifiers/priority-transfer-rollback.js \
2257
- || ! grep -Fq "assert.strictEqual(result.stderr, '')" benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/verifiers/priority-transfer-rollback.js \
2258
- || ! grep -Fq 'assert.deepStrictEqual(parsed, {' benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/verifiers/priority-transfer-rollback.js \
2259
- || grep -Fq 'parsed.applied' benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/verifiers/priority-transfer-rollback.js \
2260
- || ! grep -Fq 'On success, write exactly one JSON object to stdout and no stderr. Keys: `applied`, `rejected`, `accounts`.' benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/expected.json \
2261
- || [ "$(grep -Fc 'On success, write exactly one JSON object to stdout and no stderr. Keys: `applied`, `rejected`, `accounts`.' benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/expected.json)" -ne 1 ]; then
2262
- offenders="${offenders}"$'\n'"F31 priority verifier must bind success stderr/no-extra-output contract"
2263
- fi
2264
- if ! grep -Fq '20260512-f32-subscription-renewal-headroom' benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/NOTES.md \
2265
- || ! grep -Fq '20260512-f32-subscription-renewal-headroom' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
2266
- || ! grep -Fq '20260512-f32-subscription-renewal-headroom' benchmark/auto-resolve/README.md \
2267
- || ! grep -Fq '20260512-f32-subscription-renewal-headroom' benchmark/auto-resolve/run-real-benchmark.md; then
2268
- offenders="${offenders}"$'\n'"F32 docs must cite the measured headroom rejection before anyone counts it"
2269
- fi
2270
- if ! grep -Fq 'assert.deepEqual(output, {' benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/verifiers/priority-credit-rollback.js \
2271
- || ! grep -Fq 'Output row key names and nested `credits` key names match the visible spec exactly, with no aliased or extra keys.' benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/expected.json \
2272
- || [ "$(grep -Fc 'Output row key names and nested `credits` key names match the visible spec exactly, with no aliased or extra keys.' benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/expected.json)" -ne 1 ]; then
2273
- offenders="${offenders}"$'\n'"F32 priority verifier must own exact output-shape contract without duplicate-error overclaim"
2274
- fi
2275
- if ! grep -Fq 'retired_fixture_exists' benchmark/auto-resolve/scripts/run-headroom-candidate.sh \
2276
- || ! grep -Fq 'retired_fixture_exists' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
2277
- || ! grep -Fq 'fixture is retired and is not rerun by pair-candidate runners' benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh \
2278
- || ! grep -Fq 'fixture is retired and is not rerun by pair-candidate runners' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
2279
- || ! grep -Fq 'historical artifact replay' benchmark/auto-resolve/README.md \
2280
- || ! grep -Fq 'historical artifact replay' benchmark/auto-resolve/run-real-benchmark.md; then
2281
- offenders="${offenders}"$'\n'"pair candidate runners must reject retired fixtures explicitly and docs must reserve retired fixtures for historical artifact replay"
2282
- fi
2283
- if ! grep -Fq 'F16-cli-quote-tax-rules F23-cli-fulfillment-wave F25-cli-cart-promotion-rules' benchmark/auto-resolve/run-real-benchmark.md; then
2284
- offenders="${offenders}"$'\n'"run-real-benchmark.md examples must use the current measured F16/F23/F25 pair-evidence set"
2285
- fi
2286
- if ! grep -Fq '20260510-f16-f23-f25-combined-proof' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
2287
- || ! grep -Fq '20260510-f16-f23-f25-combined-proof' benchmark/auto-resolve/README.md \
2288
- || ! grep -Fq '20260510-f16-f23-f25-combined-proof' benchmark/auto-resolve/run-real-benchmark.md; then
2289
- offenders="${offenders}"$'\n'"benchmark docs must cite the current F16/F23/F25 pair proof run"
2290
- fi
2291
- if ! grep -Fq 'average pair margin +25.3' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
2292
- || ! grep -Fq 'average pair margin +25.3' benchmark/auto-resolve/README.md \
2293
- || ! grep -Fq 'average pair margin was `+25.3`' benchmark/auto-resolve/run-real-benchmark.md; then
2294
- offenders="${offenders}"$'\n'"benchmark docs must cite the F16/F23/F25 average pair margin"
2295
- fi
2296
- if ! grep -Fq 'average solo_claude headroom **8.0**' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
2297
- || ! grep -Fq 'minimum solo_claude' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
2298
- || ! grep -Fq '| Fixture | Bare | Solo_claude | Pair (`l2_risk_probes`) | Margin | Pair mode | Wall ratio |' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
2299
- || ! grep -Fq 'include `bare` headroom and `solo_claude` headroom' benchmark/auto-resolve/run-real-benchmark.md \
2300
- || ! grep -Fq 'default 5-point `bare`/`solo_claude` headroom margins' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
2301
- || ! grep -Fq 'average and minimum `bare`/`solo_claude`' benchmark/auto-resolve/README.md \
2302
- || ! grep -Fq 'average and minimum headroom' benchmark/auto-resolve/run-real-benchmark.md \
2303
- || ! grep -Fq 'fixture pass count' benchmark/auto-resolve/README.md \
2304
- || ! grep -Fq 'fixture pass count' benchmark/auto-resolve/run-real-benchmark.md; then
2305
- offenders="${offenders}"$'\n'"benchmark docs must cite and explain headroom set summaries"
2306
- fi
2307
- if ! grep -Fq 'both baseline arms evidence-complete' \
2308
- benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.md \
2309
- || ! grep -Fq 'both baseline arms evidence-complete' \
2310
- benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.json \
2311
- || ! grep -Fq 'Fixtures passed: 3/3 (minimum required: 3)' \
2312
- benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.md \
2313
- || ! grep -Fq 'Average solo_claude headroom: 8.0' \
2314
- benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.md \
2315
- || ! grep -Fq '"min_solo_headroom": 5' \
2316
- benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.json \
2317
- || ! grep -Fq '| Fixture | Bare | Bare headroom | Solo_claude | Solo_claude headroom | Status | Reason |' \
2318
- benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.md \
2319
- || ! grep -Fq '| F16-cli-quote-tax-rules | 50 | 10 | 75 | 5 | PASS | |' \
2320
- benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.md \
2321
- || ! grep -Fq '"solo_headroom": 14' \
2322
- benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.json; then
2323
- offenders="${offenders}"$'\n'"tracked F16/F23/F25 headroom report must use current evidence-complete wording and headroom columns"
2324
- fi
2325
- if ! grep -Fq 'l2_risk_probes evidence-clean' \
2326
- benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.md \
2327
- || ! grep -Fq 'l2_risk_probes must be evidence-clean' \
2328
- benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.json \
2329
- || ! grep -Fq 'Fixtures passed: 3/3 (minimum required: 3)' \
2330
- benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.md \
2331
- || ! grep -Fq 'Average pair margin: +25.3' \
2332
- benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.md \
2333
- || ! grep -Fq '"avg_pair_margin": 25.333333333333332' \
2334
- benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.json; then
2335
- offenders="${offenders}"$'\n'"tracked F16/F23/F25 pair report must use current l2_risk_probes evidence-clean wording and average pair margin"
2336
- fi
2337
- if ! grep -Fq 'evidence-complete `bare <= 60`' benchmark/auto-resolve/README.md \
2338
- || ! grep -Fq 'evidence-complete `bare <= 60`' benchmark/auto-resolve/run-real-benchmark.md; then
2339
- offenders="${offenders}"$'\n'"benchmark docs must describe baseline headroom arms as evidence-complete, not correctness-clean"
2340
- fi
2341
- if ! grep -Fq 'min-bare-headroom' benchmark/auto-resolve/scripts/headroom-gate.py \
2342
- || ! grep -Fq 'min-solo-headroom' benchmark/auto-resolve/scripts/headroom-gate.py \
2343
- || ! grep -Fq 'min-bare-headroom' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
2344
- || ! grep -Fq 'min-solo-headroom' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
2345
- || ! grep -Fq 'headroom >= 5' benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.md \
2346
- || ! grep -Fq 'headroom >= 5' benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.md \
2347
- || ! grep -Fq 'default minimum 5-point `bare`/`solo_claude` headroom margin' benchmark/auto-resolve/README.md \
2348
- || ! grep -Fq 'default minimum 5-point `bare`/`solo_claude` headroom margin' benchmark/auto-resolve/run-real-benchmark.md \
2349
- || ! grep -Fq 'default 5-point `bare`/`solo_claude` headroom margins' benchmark/auto-resolve/README.md \
2350
- || ! grep -Fq 'default 5-point `bare`/`solo_claude` headroom margins' benchmark/auto-resolve/BENCHMARK-DESIGN.md \
2351
- || ! grep -Fq 'bare headroom 1 < 5' benchmark/auto-resolve/scripts/test-headroom-gate.sh \
2352
- || ! grep -Fq 'bare headroom 1 < 5' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh; then
2353
- offenders="${offenders}"$'\n'"benchmark headroom gates must reject threshold-fragile candidates with default 5-point baseline headroom margins"
2354
- fi
2355
- if [ -d benchmark/auto-resolve/results/20260511-f28-headroom-smoke-085307/F28-cli-return-authorization ]; then
2356
- if python3 benchmark/auto-resolve/scripts/headroom-gate.py \
2357
- --run-id 20260511-f28-headroom-smoke-085307 \
2358
- --min-fixtures 1 \
2359
- --out-json /tmp/devlyn-lint-f28-headroom-gate.json \
2360
- --out-md /tmp/devlyn-lint-f28-headroom-gate.md >/dev/null 2>&1; then
2361
- offenders="${offenders}"$'\n'"current local F28 headroom artifact must fail the default 5-point headroom margin gate"
2362
- elif ! grep -Fq 'bare headroom 1 < 5' /tmp/devlyn-lint-f28-headroom-gate.md; then
2363
- offenders="${offenders}"$'\n'"current local F28 headroom failure must cite bare headroom 1 < 5"
2364
- fi
2365
- fi
2366
- if [ -d benchmark/auto-resolve/results/20260511-f28-pair-smoke-091021/F28-cli-return-authorization ]; then
2367
- if python3 benchmark/auto-resolve/scripts/headroom-gate.py \
2368
- --run-id 20260511-f28-pair-smoke-091021 \
2369
- --min-fixtures 1 \
2370
- --out-json /tmp/devlyn-lint-f28-pair-smoke-headroom-gate.json \
2371
- --out-md /tmp/devlyn-lint-f28-pair-smoke-headroom-gate.md >/dev/null 2>&1; then
2372
- offenders="${offenders}"$'\n'"current local F28 pair-smoke artifact must fail before pair evidence can be counted"
2373
- elif ! grep -Fq 'bare score 65 > 60' /tmp/devlyn-lint-f28-pair-smoke-headroom-gate.md; then
2374
- offenders="${offenders}"$'\n'"current local F28 pair-smoke failure must cite bare score 65 > 60"
2375
- fi
2376
- fi
2377
- if ! grep -Fq 'startup `Gate:` line' benchmark/auto-resolve/README.md \
2378
- || ! grep -Fq 'startup `Gate:` line' benchmark/auto-resolve/run-real-benchmark.md \
2379
- || ! grep -Fq 'headroom gate passed — candidate set accepted' benchmark/auto-resolve/scripts/run-headroom-candidate.sh \
2380
- || ! grep -Fq 'headroom gate failed — candidate set rejected' benchmark/auto-resolve/scripts/run-headroom-candidate.sh \
2381
- || ! grep -Fq 'candidate set was accepted or rejected' benchmark/auto-resolve/README.md \
2382
- || ! grep -Fq 'candidate set was accepted or rejected' benchmark/auto-resolve/run-real-benchmark.md \
2383
- || ! grep -Fq 'fixture score table with bare score, bare' benchmark/auto-resolve/README.md \
2384
- || ! grep -Fq 'headroom, solo_claude score, solo_claude headroom' benchmark/auto-resolve/README.md \
2385
- || ! grep -Fq 'remaining headroom against' benchmark/auto-resolve/run-real-benchmark.md \
2386
- || ! grep -Fq 'pair minus `solo_claude` margin' benchmark/auto-resolve/run-real-benchmark.md \
2387
- || ! grep -Fq 'Fixture Bare Solo_claude Pair Pair-Solo_claude' benchmark/auto-resolve/run-real-benchmark.md \
2388
- || ! grep -Fq 'fixture score table with bare,' benchmark/auto-resolve/README.md \
2389
- || ! grep -Fq 'solo_claude, pair, margin, pair-mode' benchmark/auto-resolve/README.md \
2390
- || ! grep -Fq 'pair above `solo_claude`' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
2391
- || ! grep -Fq 'Suite average variant-solo_claude margin' benchmark/auto-resolve/BENCHMARK-DESIGN.md \
2392
- || ! grep -Fq 'legacy `variant`-`bare` (L2-L0)' benchmark/auto-resolve/RUBRIC.md \
2393
- || ! grep -Fq '`solo_claude`-`bare` measures solo harness value; pair-`solo_claude` measures pair value' benchmark/auto-resolve/README.md \
2394
- || ! grep -Fq '| fixture | bare | solo_claude | solo_claude-bare |' benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh \
2395
- || ! grep -Fq '| fixture | bare | solo_claude | solo_claude-bare |' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
2396
- || ! grep -Fq '| fixture | bare | solo_claude | pair | pair-solo_claude |' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
2397
- || ! grep -Fq 'startup `Headroom:` and `Pair:` lines' benchmark/auto-resolve/run-real-benchmark.md \
2398
- || ! grep -Fq 'startup `Headroom:` / `Pair:` lines' benchmark/auto-resolve/README.md; then
2399
- offenders="${offenders}"$'\n'"benchmark docs must describe real-run startup gate lines and headroom score columns"
2400
- fi
2401
- if ! grep -Fq 'DEVLYN_BENCHMARK_CLI_SUBCOMMAND: benchmarkMode' bin/devlyn.js \
2402
- || ! grep -Fq 'npx devlyn-cli benchmark headroom --run-id "$RUN_ID"' benchmark/auto-resolve/scripts/run-headroom-candidate.sh \
2403
- || ! grep -Fq 'npx devlyn-cli benchmark pair --run-id "$RUN_ID"' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
2404
- || ! grep -Fq 'canonical trigger, margin >= +' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
2405
- || ! grep -Fq 'canonical trigger, margin >= +5' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
2406
- || ! grep -Fq 'headroom gate failed — pair arm not executed' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
2407
- || ! grep -Fq 'pair gate failed — pair evidence rejected' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
2408
- || ! grep -Fq 'headroom gate passed — executing $PAIR_ARM' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
2409
- || ! grep -Fq 'pair gate passed — pair evidence accepted' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
2410
- || ! grep -Fq 'headroom fails, the runner explicitly says the pair arm was not executed' benchmark/auto-resolve/README.md \
2411
- || ! grep -Fq 'pair gate fails, it explicitly says pair evidence was rejected' benchmark/auto-resolve/README.md \
2412
- || ! grep -Fq 'then that pair evidence was accepted' benchmark/auto-resolve/README.md \
2413
- || ! grep -Fq 'If headroom fails, it reports that the pair arm was not executed' benchmark/auto-resolve/run-real-benchmark.md \
2414
- || ! grep -Fq 'pair gate fails, it reports that pair evidence was rejected' benchmark/auto-resolve/run-real-benchmark.md \
2415
- || ! grep -Fq 'then that pair evidence was' benchmark/auto-resolve/run-real-benchmark.md \
2416
- || ! grep -Fq 'accepted. When launched through' benchmark/auto-resolve/run-real-benchmark.md \
2417
- || ! grep -Fq 'the replay `Command:` uses the' benchmark/auto-resolve/README.md \
2418
- || ! grep -Fq 'same package CLI path' benchmark/auto-resolve/README.md \
2419
- || ! grep -Fq 'the replay command uses' benchmark/auto-resolve/run-real-benchmark.md \
2420
- || ! grep -Fq 'uses that same' benchmark/auto-resolve/run-real-benchmark.md \
2421
- || ! grep -Fq 'package CLI path' benchmark/auto-resolve/run-real-benchmark.md; then
2422
- offenders="${offenders}"$'\n'"benchmark CLI headroom/pair runs must replay as npx devlyn-cli commands and docs must state that"
2423
- fi
2424
- if ! grep -Fq 'npx devlyn-cli benchmark headroom --min-fixtures 3 F16-cli-quote-tax-rules F23-cli-fulfillment-wave F25-cli-cart-promotion-rules' README.md \
2425
- || ! grep -Fq 'npx devlyn-cli benchmark frontier --out-md /tmp/devlyn-pair-frontier.md' README.md \
2426
- || ! grep -Fq 'npx devlyn-cli benchmark pair --min-fixtures 3 --max-pair-solo-wall-ratio 3 F16-cli-quote-tax-rules F23-cli-fulfillment-wave F25-cli-cart-promotion-rules' README.md \
2427
- || ! grep -Fq 'average pair margin' README.md \
2428
- || ! grep -Fq 'default 5-point `bare`/`solo_claude` headroom margins' README.md \
2429
- || ! grep -Fq 'Add `--dry-run` to either score runner' README.md \
2430
- || ! grep -Fq 'fixture count, and the replay command' README.md \
2431
- || ! grep -Fq 'Dry-runs' README.md \
2432
- || ! grep -Fq 'and lint prove wiring only; real score claims must cite the run id and fixture' README.md; then
2433
- offenders="${offenders}"$'\n'"README.md must expose score-focused benchmark headroom/pair CLI paths and distinguish wiring checks from real scores"
2434
- fi
2435
- if ! grep -Fq 'npx devlyn-cli benchmark pair --min-fixtures 3 --max-pair-solo-wall-ratio 3 F16-cli-quote-tax-rules F23-cli-fulfillment-wave F25-cli-cart-promotion-rules' bin/devlyn.js \
2436
- || ! grep -Fq -- '--max-pair-solo-wall-ratio N default: 3' bin/devlyn.js \
2437
- || ! grep -Fq 'MAX_PAIR_SOLO_WALL_RATIO=3' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
2438
- || ! grep -Fq 'parser.add_argument("--max-pair-solo-wall-ratio", type=positive_float, default=3.0)' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
2439
- || ! grep -Fq '"max_observed_pair_solo_wall_ratio": max(ratios) if ratios else None' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
2440
- || ! grep -Fq 'Allowed pair/solo wall ratio' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
2441
- || ! grep -Fq 'Maximum observed pair/solo wall ratio' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
2442
- || ! grep -Fq 'Maximum observed pair/solo wall ratio: 2.00x' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
2443
- || ! grep -Fq 'separates the allowed pair/solo wall ratio from the maximum observed pair/solo wall ratio' benchmark/auto-resolve/run-real-benchmark.md \
2444
- || ! grep -Fq 'python3 "$GATE" --results-root "$TMP_DIR" --run-id slow-pair' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
2445
- || ! grep -Fq -- '--min-fixtures 3' benchmark/auto-resolve/run-real-benchmark.md \
2446
- || ! grep -Fq -- '--min-fixtures 3' benchmark/auto-resolve/README.md \
2447
- || ! grep -Fq 'average pair margin' benchmark/auto-resolve/run-real-benchmark.md \
2448
- || ! grep -Fq 'average pair margin' benchmark/auto-resolve/README.md \
2449
- || ! grep -Fq -- '--dry-run validate args/fixtures and print replay command only' bin/devlyn.js \
2450
- || ! grep -Fq 'does not produce' benchmark/auto-resolve/run-real-benchmark.md \
2451
- || ! grep -Fq 'scores. When showing' benchmark/auto-resolve/run-real-benchmark.md; then
2452
- offenders="${offenders}"$'\n'"benchmark pair examples must explicitly gate the current F16/F23/F25 proof set with --min-fixtures 3"
2453
- fi
2454
- if ! grep -Fq 'use 3 for F16/F23/F25 proof reruns; audit requires 4 passing evidence rows' bin/devlyn.js \
2455
- || ! grep -Fq 'use 3 for F16/F23/F25 proof reruns; audit requires 4 passing evidence rows' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh; then
2456
- offenders="${offenders}"$'\n'"benchmark CLI help must distinguish proof reruns from the four-row release audit"
2457
- fi
2458
- if ! grep -Fq 'judge blind mapping missing' benchmark/auto-resolve/scripts/headroom-gate.py \
2459
- || ! grep -Fq 'judge blind mapping missing' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
2460
- || ! grep -Fq 'blind_mapping_arm_missing' benchmark/auto-resolve/scripts/compile-report.py \
2461
- || ! grep -Fq 'def trusted_winner()' benchmark/auto-resolve/scripts/compile-report.py \
2462
- || ! grep -Fq 'raw_findings_by_arm = judge.get("findings_by_arm")' benchmark/auto-resolve/scripts/compile-report.py \
2463
- || ! grep -Fq 'def critical_findings_for(arm: str)' benchmark/auto-resolve/scripts/compile-report.py \
2464
- || ! grep -Fq 'def exact_bool(value)' benchmark/auto-resolve/scripts/compile-report.py \
2465
- || ! grep -Fq '"malformed_boolean_fields": malformed_boolean_fields' benchmark/auto-resolve/scripts/compile-report.py \
2466
- || ! grep -Fq '"dq_judge_malformed": judge_dq_malformed' benchmark/auto-resolve/scripts/compile-report.py \
2467
- || ! grep -Fq 'def bool_or_none(value)' benchmark/auto-resolve/scripts/ship-gate.py \
2468
- || ! grep -Fq 'l1_dq_by_fixture: dict[str, bool]' benchmark/auto-resolve/scripts/ship-gate.py \
2469
- || ! grep -Fq 'summary arms_present malformed' benchmark/auto-resolve/scripts/ship-gate.py \
2470
- || ! grep -Fq 'summary arms_present.solo_claude malformed' benchmark/auto-resolve/scripts/ship-gate.py \
2471
- || ! grep -Fq 'summary margins_avg malformed' benchmark/auto-resolve/scripts/ship-gate.py \
2472
- || ! grep -Fq 'raw_findings_letters = chosen.get("critical_findings")' benchmark/auto-resolve/scripts/judge.sh \
2473
- || ! grep -Fq 'raw_findings_letters = chosen.get("critical_findings")' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
2474
- || ! grep -Fq 'raw_validation = judge.get("_axis_validation")' benchmark/auto-resolve/scripts/compile-report.py \
2475
- || ! grep -Fq 'raw_validation = judge.get("_axis_validation")' benchmark/auto-resolve/scripts/headroom-gate.py \
2476
- || ! grep -Fq 'raw_validation = judge.get("_axis_validation")' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
2477
- || ! grep -Fq 'raw_scores_by_arm = judge.get("scores_by_arm")' benchmark/auto-resolve/scripts/compile-report.py \
2478
- || ! grep -Fq 'from pair_evidence_contract import is_score, is_strict_number' benchmark/auto-resolve/scripts/compile-report.py \
2479
- || ! grep -Fq 'loads_strict_json_object(path.read_text())' benchmark/auto-resolve/scripts/compile-report.py \
2480
- || ! grep -Fq 'parse_constant=reject_json_constant' benchmark/auto-resolve/scripts/ship-gate.py \
2481
- || ! grep -Fq 'if arm in mapped_arms and is_score(score)' benchmark/auto-resolve/scripts/compile-report.py \
2482
- || ! grep -Fq 'def strict_number(value)' benchmark/auto-resolve/scripts/compile-report.py \
2483
- || ! grep -Fq 'raw_scores = judge.get("scores_by_arm")' benchmark/auto-resolve/scripts/headroom-gate.py \
2484
- || ! grep -Fq 'raw_scores = judge.get("scores_by_arm")' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
2485
- || ! grep -Fq 'from pair_evidence_contract import is_score' benchmark/auto-resolve/scripts/headroom-gate.py \
2486
- || ! grep -Fq 'return value if is_score(value) else None' benchmark/auto-resolve/scripts/headroom-gate.py \
2487
- || ! grep -Fq 'raw_scores = judge.get("scores_by_arm")' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
2488
- || ! grep -Fq 'from pair_evidence_contract import is_score' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
2489
- || ! grep -Fq 'def exact_bool(value: object)' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
2490
- || ! grep -Fq 'def disqualifier_flag(value: object' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
2491
- || ! grep -Fq 'if not is_score(solo) or not is_score(bare):' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
2492
- || ! grep -Fq 'if is_score(score):' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
2493
- || ! grep -Fq 'return legacy if is_score(legacy) else None' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
2494
- || ! grep -Fq 'raw_scores = judge.get("scores_by_arm")' benchmark/auto-resolve/scripts/iter-0033c-l1-summary.py \
2495
- || ! grep -Fq 'raw_scores = judge.get("scores_by_arm")' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
2496
- || ! grep -Fq 'from pair_evidence_contract import (' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
2497
- || ! grep -Fq 'is_score,' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
2498
- || ! grep -Fq 'is_strict_number,' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
2499
- || ! grep -Fq 'loads_strict_json_object(p.read_text())' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
2500
- || ! grep -Fq 'parse_constant=reject_json_constant' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
2501
- || ! grep -Fq 'PAIR_VERDICTS = {"PASS", "PASS_WITH_ISSUES", "NEEDS_WORK", "BLOCKED", "FAIL"}' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
2502
- || ! grep -Fq 'def is_pair_judge_verdict(value: object) -> bool:' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
2503
- || ! grep -Fq 'def exact_bool(value: object)' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
2504
- || ! grep -Fq 'def bool_flag(value: object' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
2505
- || ! grep -Fq 'if is_score(sba.get(arm)):' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
2506
- || ! grep -Fq 'return legacy if is_score(legacy) else None' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
2507
- || ! grep -Fq 'def strict_elapsed_seconds' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
2508
- || ! grep -Fq 'def timeout_flag(result: dict | None) -> bool:' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
2509
- || ! grep -Fq 'raw_scores = judge.get("scores_by_arm")' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
2510
- || ! grep -Fq 'raw_dq_by_arm = judge.get("disqualifiers_by_arm")' benchmark/auto-resolve/scripts/headroom-gate.py \
2511
- || ! grep -Fq 'raw_dq_by_arm = judge.get("disqualifiers_by_arm")' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
2512
- || ! grep -Fq 'raw_dq_by_arm = judge.get("disqualifiers_by_arm")' benchmark/auto-resolve/scripts/compile-report.py \
2513
- || ! grep -Fq 'raw_by_arm = judge.get("disqualifiers_by_arm")' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
2514
- || ! grep -Fq 'raw_dba = judge.get("disqualifiers_by_arm")' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
2515
- || ! grep -Fq 'raw_legacy = judge.get("disqualifiers")' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
2516
- || ! grep -Fq 'raw_dqs = judge.get("disqualifiers")' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
2517
- || ! grep -Fq 'def is_score(value):' benchmark/auto-resolve/scripts/judge.sh \
2518
- || ! grep -Fq 'invalid judge score value(s)' benchmark/auto-resolve/scripts/judge.sh \
2519
- || ! grep -Fq 'invalid judge disqualifier value(s)' benchmark/auto-resolve/scripts/judge.sh \
2520
- || ! grep -Fq 'invalid opus disqualifier value(s)' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
2521
- || ! grep -Fq 'if arm is not None and key in chosen and is_score(chosen[key]):' benchmark/auto-resolve/scripts/judge.sh \
2522
- || ! grep -Fq 'raw_dq_letters = chosen.get("disqualifiers")' benchmark/auto-resolve/scripts/judge.sh \
2523
- || ! grep -Fq 'raw_dq_letters = chosen.get("disqualifiers")' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
2524
- || ! grep -Fq 'scores_by_arm` alone is not evidence' benchmark/auto-resolve/README.md \
2525
- || ! grep -Fq 'without the blind slot mapping is not score evidence' benchmark/auto-resolve/run-real-benchmark.md \
2526
- || ! grep -Fq 'absent from the blind mapping is not score evidence' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
2527
- || ! grep -Fq 'wrong-pair-mapping' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
2528
- || ! grep -Fq 'from pair_evidence_contract import (' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
2529
- || ! grep -Fq 'ALLOWED_PAIR_ARMS,' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
2530
- || ! grep -Fq 'loads_strict_json_object,' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
2531
- || ! grep -Fq 'def bool_flag_failure(value: Any, true_reason: str, malformed_reason: str) -> str | None:' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
2532
- || ! grep -Fq 'def bool_flag_failure(value: object, true_reason: str, malformed_reason: str) -> str | None:' benchmark/auto-resolve/scripts/headroom-gate.py \
2533
- || ! grep -Fq 'return value if is_score(value) else None' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
2534
- || ! grep -Fq 'if not is_strict_number(pair_elapsed) or not is_strict_number(solo_elapsed):' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
2535
- || ! grep -Fq 'return is_strict_number(value) and value >= 1.0' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
2536
- || ! grep -Fq 'value must be finite and > 0' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
2537
- || ! grep -Fq 'overrange-score' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
2538
- || ! grep -Fq 'boolean-score' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
2539
- || ! grep -Fq 'boolean-wall-time' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
2540
- || ! grep -Fq 'invalid-max-wall-ratio' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
2541
- || ! grep -Fq 'boolean-pair-verify-score' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
2542
- || ! grep -Fq 'malformed-pair-bool' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
2543
- || ! grep -Fq 'malformed-judge-bool' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
2544
- || ! grep -Fq 'wrong-mapping' benchmark/auto-resolve/scripts/test-headroom-gate.sh \
2545
- || ! grep -Fq 'overrange-score' benchmark/auto-resolve/scripts/test-headroom-gate.sh \
2546
- || ! grep -Fq 'boolean-score' benchmark/auto-resolve/scripts/test-headroom-gate.sh \
2547
- || ! grep -Fq 'malformed-bare-bool' benchmark/auto-resolve/scripts/test-headroom-gate.sh \
2548
- || ! grep -Fq 'bare judge disqualifier malformed' benchmark/auto-resolve/scripts/test-headroom-gate.sh \
2549
- || ! grep -Fq 'variant-mapping-disqualifies' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2550
- || ! grep -Fq 'solo-mapping-disqualifies' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2551
- || ! grep -Fq 'stale-margin' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2552
- || ! grep -Fq 'stale judge margins must be recomputed from trusted scores' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2553
- || ! grep -Fq 'winner without blind-mapped trusted score must be null' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2554
- || ! grep -Fq 'malformed scores_by_arm must not expose' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2555
- || ! grep -Fq 'out-of-range scores_by_arm must not expose variant score' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2556
- || ! grep -Fq 'boolean scores_by_arm must not expose solo score' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2557
- || ! grep -Fq 'boolean result numeric fields must not appear in compile summary' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2558
- || ! grep -Fq 'malformed-result-bool' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2559
- || ! grep -Fq 'malformed-judge-bool' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2560
- || ! grep -Fq 'malformed-l1-dq-summary' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2561
- || ! grep -Fq 'malformed-arms-present-wrapper' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2562
- || ! grep -Fq 'malformed-arms-present' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2563
- || ! grep -Fq 'malformed-margins-avg-wrapper' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2564
- || ! grep -Fq 'F1 L1 disqualifier malformed' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2565
- || ! grep -Fq 'overrange C1 row fields must not promote F17' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
2566
- || ! grep -Fq 'string C1 disqualifier must not promote F18' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
2567
- || ! grep -Fq 'f9-overrange-scores' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
2568
- || ! grep -Fq 'f9-boolean-scores' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
2569
- || ! grep -Fq 'f9-string-dq-entry' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
2570
- || ! grep -Fq 'overrange-scores-results' benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh \
2571
- || ! grep -Fq 'boolean-scores-results' benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh \
2572
- || ! grep -Fq 'boolean-wall-results' benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh \
2573
- || ! grep -Fq 'nan-wall-results' benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh \
2574
- || ! grep -Fq 'string-timeout-results' benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh \
2575
- || ! grep -Fq 'malformed-pair-state-results' benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh \
2576
- || ! grep -Fq 'non-list finding entry must become a one-item list' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2577
- || ! grep -Fq 'non-dict findings_by_arm must be ignored' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2578
- || ! grep -Fq 'non-dict _axis_validation wrapper must not crash' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2579
- || ! grep -Fq 'load_dict_json' benchmark/auto-resolve/scripts/compile-report.py \
2580
- || ! grep -Fq 'measurement invalid: malformed summary.json' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2581
- || ! grep -Fq 'nan-summary' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2582
- || ! grep -Fq 'summary rows contain non-object entries' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2583
- || ! grep -Fq 'malformed-summary-field-types' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2584
- || ! grep -Fq 'variant axis count malformed' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2585
- || ! grep -Fq 'result.json malformed' benchmark/auto-resolve/scripts/test-headroom-gate.sh \
2586
- || ! grep -Fq 'result.json malformed' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
2587
- || ! grep -Fq 'non-dict variant result.json must fail closed' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2588
- || ! grep -Fq 'NaN variant result.json must fail closed as a disqualifier' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2589
- || ! grep -Fq '("solo_claude (L1)", "solo_claude")' benchmark/auto-resolve/scripts/compile-report.py \
2590
- || ! grep -Fq '| Fixture | Category | variant (L2) | solo_claude (L1) | bare (L0) | variant-bare | solo_claude-bare | variant-solo_claude | Winner | Wall variant/solo_claude/bare | Wall variant/solo_claude | Wall variant/bare |' benchmark/auto-resolve/scripts/compile-report.py \
2591
- || ! grep -Fq '| Fixture | Category | variant (L2) | solo_claude (L1) | bare (L0) | variant-bare | solo_claude-bare | variant-solo_claude | Winner | Wall variant/solo_claude/bare | Wall variant/solo_claude | Wall variant/bare |' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2592
- || ! grep -Fq '**solo_claude (L1):**' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2593
- || ! grep -Fq 'variant (L2) vs solo_claude (L1) margin avg' benchmark/auto-resolve/scripts/compile-report.py \
2594
- || ! grep -Fq '**variant (L2) vs solo_claude (L1) margin avg:** +10.0' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2595
- || ! grep -Fq 'Wall ratio variant (L2) / solo_claude (L1)' benchmark/auto-resolve/scripts/compile-report.py \
2596
- || ! grep -Fq '**Wall ratio variant (L2) / solo_claude (L1):** 1.0x' benchmark/auto-resolve/scripts/test-ship-gate.sh \
2597
- || ! grep -Fq 'malformed-scores' benchmark/auto-resolve/scripts/test-headroom-gate.sh \
2598
- || ! grep -Fq 'malformed-scores' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
2599
- || ! grep -Fq 'malformed-dq-entry' benchmark/auto-resolve/scripts/test-headroom-gate.sh \
2600
- || ! grep -Fq 'malformed-dq-entry' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
2601
- || ! grep -Fq 'malformed-axis-wrapper' benchmark/auto-resolve/scripts/test-headroom-gate.sh \
2602
- || ! grep -Fq 'malformed-axis-wrapper' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
2603
- || ! grep -Fq 'fixture-nan-metadata' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
2604
- || ! grep -Fq 'fixture-nan-expected' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
2605
- || ! grep -Fq 'c1-summary malformed: expected object' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
2606
- || ! grep -Fq 'f9-judge malformed: expected object' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
2607
- || ! grep -Fq 'malformed C1 row fields must not promote F1' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
2608
- || ! grep -Fq 'F9 must not be included when scores_by_arm is malformed' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
2609
- || ! grep -Fq 'truthy malformed disqualifier entry must exclude F9' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
2610
- || ! grep -Fq 'disqualifiers": ["not", "a", "dict"]' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
2611
- || ! grep -Fq 'bad-mapping' benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh \
2612
- || ! grep -Fq 'write_fixture_with_malformed_dq_entry' benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh \
2613
- || ! grep -Fq 'write_fixture_with_malformed_legacy_dq' benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh \
2614
- || ! grep -Fq 'write_fixture_with_string_dq_entry' benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh \
2615
- || ! grep -Fq 'write_fixture_with_malformed_result' benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh \
2616
- || ! grep -Fq 'disqualifiers": ["not", "a", "dict"]' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
2617
- || ! grep -Fq 'malformed compare.json for malformed-compare' benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh \
2618
- || ! grep -Fq 'malformed compare.json for nan-compare: invalid JSON' benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh \
2619
- || ! grep -Fq 'malformed-compare-sections' benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh \
2620
- || ! grep -Fq 'malformed-verdict-fields' benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh \
2621
- || ! grep -Fq 'malformed-elapsed-fields' benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh \
2622
- || ! grep -Fq 'string-pair-mode' benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh \
2623
- || ! grep -Fq 'def strict_positive_number(value):' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
2624
- || ! grep -Fq 'math.isfinite(value)' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
2625
- || ! grep -Fq 'def strict_nonnegative_int(value):' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
2626
- || ! grep -Fq 'def summary_findings_count(data):' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
2627
- || ! grep -Fq 'def severity_count_sum(data):' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
2628
- || ! grep -Fq 'def strict_greater(left, right):' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
2629
- || ! grep -Fq 'metadata = loads_strict_json_object(pathlib.Path(sys.argv[1]).read_text())' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
2630
- || ! grep -Fq 'metadata timeout_seconds must be a positive integer' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
2631
- || ! grep -Fq 'state = as_dict(loads_strict_json_object(state_path.read_text())) if state_path.is_file() else {}' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
2632
- || ! grep -Fq 'json.loads(line, parse_constant=reject_json_constant)' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
2633
- || ! grep -Fq 'summary = loads_strict_json_object(pathlib.Path(sys.argv[1]).read_text())' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
2634
- || ! grep -Fq 'expected = loads_strict_json_object(pathlib.Path(sys.argv[1]).read_text())' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
2635
- || ! grep -Fq 'out[arm] = loads_strict_json_object(path.read_text()) if path.is_file() else {"missing": True}' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
2636
- || ! grep -Fq 'def as_dict(value):' benchmark/auto-resolve/scripts/run-fixture.sh \
2637
- || ! grep -Fq 'metadata = loads_strict_json_object(pathlib.Path(sys.argv[1]).read_text())' benchmark/auto-resolve/scripts/run-fixture.sh \
2638
- || ! grep -Fq 'metadata timeout_seconds must be a positive integer' benchmark/auto-resolve/scripts/run-fixture.sh \
2639
- || ! grep -Fq 'expected = loads_strict_json_object(pathlib.Path(sys.argv[1]).read_text())' benchmark/auto-resolve/scripts/run-fixture.sh \
2640
- || ! grep -Fq 'verify = as_dict(loads_strict_json_object(pathlib.Path(result_dir, "verify.json").read_text()))' benchmark/auto-resolve/scripts/run-fixture.sh \
2641
- || ! grep -Fq 'loads_strict_json_object(pathlib.Path(result_dir, "timing.json").read_text())' benchmark/auto-resolve/scripts/run-fixture.sh \
2642
- || ! grep -Fq 'loads_strict_json_object(pathlib.Path(state_path).read_text())' benchmark/auto-resolve/scripts/run-fixture.sh \
2643
- || ! grep -Fq 'phases = as_dict(state.get("phases"))' benchmark/auto-resolve/scripts/run-fixture.sh \
2644
- || ! grep -Fq 'legacy_verify = as_dict(state.get("verify"))' benchmark/auto-resolve/scripts/run-fixture.sh \
2645
- || ! grep -Fq 'data = raw_oracle' benchmark/auto-resolve/scripts/run-fixture.sh \
2646
- || ! grep -Fq 'oracle artifact malformed or unreadable' benchmark/auto-resolve/scripts/run-fixture.sh \
2647
- || ! grep -Fq '"type": "oracle-error"' benchmark/auto-resolve/scripts/run-fixture.sh \
2648
- || ! grep -Fq 'verify["oracle_disqualifier"] = True' benchmark/auto-resolve/scripts/run-fixture.sh \
2649
- || ! grep -Fq 'findings = raw_findings if isinstance(raw_findings, list) else []' benchmark/auto-resolve/scripts/run-fixture.sh \
2650
- || ! grep -Fq 'if not isinstance(finding, dict):' benchmark/auto-resolve/scripts/run-fixture.sh \
2651
- || ! grep -Fq 'def as_dict(value):' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
2652
- || ! grep -Fq 'phases = as_dict(state.get("phases"))' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
2653
- || ! grep -Fq 'legacy_verify = as_dict(state.get("verify"))' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
2654
- || ! grep -Fq 'PAIR_VERDICTS = {"PASS", "PASS_WITH_ISSUES", "NEEDS_WORK", "BLOCKED", "FAIL"}' benchmark/auto-resolve/scripts/run-fixture.sh \
2655
- || ! grep -Fq 'def has_pair_judge_verdict(sub_verdicts):' benchmark/auto-resolve/scripts/run-fixture.sh \
2656
- || ! grep -Fq 'PAIR_VERDICTS = {"PASS", "PASS_WITH_ISSUES", "NEEDS_WORK", "BLOCKED", "FAIL"}' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
2657
- || ! grep -Fq 'def has_pair_judge_verdict(sub_verdicts):' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
2658
- || ! grep -Fq 'or verify_phase.get("pair_mode") is True' benchmark/auto-resolve/scripts/run-fixture.sh \
2659
- || ! grep -Fq 'has_pair_judge_verdict(sub_verdicts) or verify_phase.get("pair_mode") is True' benchmark/auto-resolve/scripts/run-fixture.sh \
2660
- || ! grep -Fq 'has_pair_judge_verdict(sub_verdicts)' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
2661
- || ! grep -Fq 'raw_pair_sub = pair.get("sub_verdicts")' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
2662
- || ! grep -Fq 'raw_pair_trigger = pair.get("pair_trigger")' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
2663
- || ! grep -Fq 'pair_trigger = raw_pair_trigger if isinstance(raw_pair_trigger, dict) else {}' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
2664
- || ! grep -Fq 'pair_mode_true = pair.get("pair_mode") is True' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
2665
- || ! grep -Fq 'def fmt_trigger_reasons(value):' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
2666
- || ! grep -Fq "| Arm | Verdict | Pair mode | Triggers | Findings | Elapsed | Invoke exit | Failure |" benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
2667
- || ! grep -Fq '| Arm | Verdict | Pair mode | Triggers | Findings | Elapsed | Invoke exit | Failure |' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2668
- || ! grep -Fq 'pair_mode = pair.get("pair_mode") is True' benchmark/auto-resolve/scripts/frozen-verify-gate.py \
2669
- || ! grep -Fq 'parse_constant=reject_json_constant' benchmark/auto-resolve/scripts/frozen-verify-gate.py \
2670
- || ! grep -Fq 'loads_strict_json_object(path.read_text())' benchmark/auto-resolve/scripts/check-f9-artifacts.py \
2671
- || ! grep -Fq 'invalid JSON numeric constant: NaN' benchmark/auto-resolve/scripts/test-check-f9-artifacts.sh \
2672
- || ! grep -Fq 'loads_strict_json_object(path.read_text(encoding="utf8"))' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
2673
- || ! grep -Fq 'def pair_trigger_failures(' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
2674
- || ! grep -Fq 'def pair_trigger_reasons(pair: dict[str, Any]) -> list[str]:' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
2675
- || ! grep -Fq 'pair_trigger_eligible(pair)' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
2676
- || ! grep -Fq '"pair_trigger_has_canonical_reason": has_canonical_pair_trigger_reason(trigger_reasons)' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
2677
- || ! grep -Fq '"pair_trigger_has_canonical_reason": true' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2678
- || ! grep -Fq 'failed attempt: pair trigger contract: ' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
2679
- || ! grep -Fq 'has_known_pair_trigger_reason(reasons)' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
2680
- || ! grep -Fq 'all_known_pair_trigger_reasons(trigger["reasons"])' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
2681
- || ! grep -Fq 'has_canonical_pair_trigger_reason(trigger["reasons"])' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
2682
- || ! grep -Fq 'path_has_actionable_solo_headroom_hypothesis(fixture_spec)' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
2683
- || ! grep -Fq 'pair_trigger reasons missing known trigger reason' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
2684
- || ! grep -Fq 'pair_trigger reasons contain unknown trigger reason' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
2685
- || ! grep -Fq 'pair_trigger reasons missing canonical trigger reason' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
2686
- || ! grep -Fq 'pair_trigger missing spec.solo_headroom_hypothesis' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
2687
- || ! grep -Fq -- '--require-hypothesis-trigger requires --fixtures-root' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
2688
- || ! grep -Fq 'swebench-missing-hypothesis-trigger-test' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2689
- || ! grep -Fq -- '--require-hypothesis-trigger requires --fixtures-root' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2690
- || ! grep -Fq 'has_known_pair_trigger_reason(reasons)' benchmark/auto-resolve/scripts/frozen-verify-gate.py \
2691
- || ! grep -Fq 'all_known_pair_trigger_reasons(reasons)' benchmark/auto-resolve/scripts/frozen-verify-gate.py \
2692
- || ! grep -Fq 'has_canonical_pair_trigger_reason(reasons)' benchmark/auto-resolve/scripts/frozen-verify-gate.py \
2693
- || ! grep -Fq 'def pair_trigger_reasons(pair: dict[str, Any]) -> list[str]:' benchmark/auto-resolve/scripts/frozen-verify-gate.py \
2694
- || ! grep -Fq 'path_has_actionable_solo_headroom_hypothesis(fixtures_root / fixture_id / "spec.md")' benchmark/auto-resolve/scripts/frozen-verify-gate.py \
2695
- || ! grep -Fq 'pair_trigger missing spec.solo_headroom_hypothesis' benchmark/auto-resolve/scripts/frozen-verify-gate.py \
2696
- || ! grep -Fq -- '--require-hypothesis-trigger' benchmark/auto-resolve/scripts/frozen-verify-gate.py \
2697
- || ! grep -Fq -- '--require-hypothesis-trigger' benchmark/auto-resolve/scripts/run-swebench-frozen-corpus.sh \
2698
- || ! grep -Fq 'missing-hypothesis-trigger' benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh \
2699
- || ! grep -Fq '"pair_trigger_has_canonical_reason": has_canonical_pair_trigger_reason(trigger_reasons)' benchmark/auto-resolve/scripts/frozen-verify-gate.py \
2700
- || ! grep -Fq '"pair_trigger_has_canonical_reason": true' benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh \
2701
- || ! grep -Fq '| Run | Fixture | Solo VERIFY | Pair VERIFY | Pair mode | Triggers | Wall ratio | External lift | Internal lift | Status | Reason |' benchmark/auto-resolve/scripts/frozen-verify-gate.py \
2702
- || ! grep -Fq '| Run | Fixture | Solo VERIFY | Pair VERIFY | Pair mode | Triggers | Wall ratio | External lift | Internal lift | Status | Reason |' benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh \
2703
- || ! grep -Fq 'pair_trigger reasons missing known trigger reason' benchmark/auto-resolve/scripts/frozen-verify-gate.py \
2704
- || ! grep -Fq 'pair_trigger reasons contain unknown trigger reason' benchmark/auto-resolve/scripts/frozen-verify-gate.py \
2705
- || ! grep -Fq 'pair_trigger reasons missing canonical trigger reason' benchmark/auto-resolve/scripts/frozen-verify-gate.py \
2706
- || ! grep -Fq 'normalized-canonical-pair-trigger' benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh \
2707
- || ! grep -Fq '| Fixture | Solo VERIFY | Pair VERIFY | Pair mode | Pair trigger | Triggers | Wall ratio | External lift | Internal lift | Included | Classification |' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
2708
- || ! grep -Fq '| Fixture | Solo VERIFY | Pair VERIFY | Pair mode | Pair trigger | Triggers | Wall ratio | External lift | Internal lift | Included | Classification |' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2709
- || ! grep -Fq 'def is_true(value: Any) -> bool:' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
2710
- || ! grep -Fq 'swebench-bool-elapsed-test' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2711
- || ! grep -Fq 'runner-nan-metadata' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2712
- || ! grep -Fq 'runner-nan-expected' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2713
- || ! grep -Fq 'json.loads(line, parse_constant=reject_json_constant)' benchmark/auto-resolve/scripts/run-swebench-solver-batch.sh \
2714
- || ! grep -Fq 'nan-instance-row' benchmark/auto-resolve/scripts/test-run-swebench-solver-batch.sh \
2715
- || ! grep -Fq 'manifest = loads_strict_json_object(pathlib.Path(sys.argv[1]).read_text())' benchmark/auto-resolve/scripts/run-iter-0033c.sh \
2716
- || ! grep -Fq 'manifest fixtures_pair_eligible must be a string array' benchmark/auto-resolve/scripts/run-iter-0033c.sh \
2717
- || ! grep -Fq 'parsed = json.loads(ln, parse_constant=reject_json_constant)' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
2718
- || ! grep -Fq 'data = loads_strict_json_object(path.read_text())' benchmark/auto-resolve/scripts/judge.sh \
2719
- || ! grep -Fq 'decoder = json.JSONDecoder(parse_constant=reject_json_constant)' benchmark/auto-resolve/scripts/judge.sh \
2720
- || ! grep -Fq 'judge = loads_strict_json_object(judge_path.read_text())' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
2721
- || ! grep -Fq 'gpt = loads_strict_json_object(pathlib.Path(sys.argv[2]).read_text())' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
2722
- || ! grep -Fq 'decoder = json.JSONDecoder(parse_constant=reject_json_constant)' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
2723
- || ! grep -Fq 'g = loads_strict_json_object(g_f.read_text())' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
2724
- || ! grep -Fq 'swebench-malformed-pair-judge-test' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2725
- || ! grep -Fq 'swebench-malformed-pair-trigger-test' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2726
- || ! grep -Fq 'swebench-unknown-pair-trigger-test' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2727
- || ! grep -Fq 'swebench-normalized-pair-trigger-test' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2728
- || ! grep -Fq 'swebench-mixed-unknown-pair-trigger-test' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2729
- || ! grep -Fq '"classification": "failed attempt: pair trigger contract: pair_trigger missing or malformed"' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2730
- || ! grep -Fq '"classification": "failed attempt: pair trigger contract: pair_trigger reasons missing known trigger reason"' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2731
- || ! grep -Fq '"classification": "failed attempt: pair trigger contract: pair_trigger reasons contain unknown trigger reason"' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2732
- || ! grep -Fq '"classification": "failed attempt: pair trigger contract: pair_trigger reasons missing canonical trigger reason"' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2733
- || ! grep -Fq 'historical-only-pair-trigger' benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh \
2734
- || ! grep -Fq 'HISTORICAL_ONLY_TRIGGER_RUN_ID' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2735
- || ! grep -Fq '| Pair trigger |' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2736
- || ! grep -Fq '"verify_findings_count": "2"' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2737
- || ! grep -Fq '"pair_found_more_low_or_worse": false' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2738
- || ! grep -Fq 'pair-trigger eligibility/contract failures' benchmark/auto-resolve/README.md \
2739
- || ! grep -Fq 'swebench-string-bool-matrix-test' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2740
- || ! grep -Fq 'findings = raw_findings if isinstance(raw_findings, list) else []' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
2741
- || ! grep -Fq 'oracle artifact malformed or unreadable' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
2742
- || ! grep -Fq 'scope-tier-a-nan' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
2743
- || ! grep -Fq 'scope-tier-b-nan' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
2744
- || ! grep -Fq 'expected.json malformed: spec_output_files must be a string array' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
2745
- || ! grep -Fq 'loads_strict_json_object(exp_path.read_text())' benchmark/auto-resolve/scripts/oracle-scope-tier-a.py \
2746
- || ! grep -Fq 'loads_strict_json_object(pathlib.Path(args.expected).read_text())' benchmark/auto-resolve/scripts/oracle-scope-tier-b.py \
2747
- || ! grep -Fq 'failed attempt: malformed compare' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2748
- || ! grep -Fq 'swebench-nan-matrix-test' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2749
- || ! grep -Fq 'parse_prepared_case' benchmark/auto-resolve/scripts/prepare-swebench-frozen-corpus.py \
2750
- || ! grep -Fq 'loads_strict_json_object(path.read_text(encoding="utf8"))' benchmark/auto-resolve/scripts/prepare-swebench-frozen-case.py \
2751
- || ! grep -Fq 'parse_constant=reject_json_constant' benchmark/auto-resolve/scripts/collect-swebench-predictions.py \
2752
- || ! grep -Fq 'parse_constant=reject_json_constant' benchmark/auto-resolve/scripts/prepare-swebench-frozen-corpus.py \
2753
- || ! grep -Fq 'loads_strict_json_object(stdout)' benchmark/auto-resolve/scripts/prepare-swebench-frozen-corpus.py \
2754
- || ! grep -Fq 'batch-bad-timeout' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2755
- || ! grep -Fq 'prepare-bad-timeout' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2756
- || ! grep -Fq 'prepare-nan-case' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2757
- || ! grep -Fq 'missing patch.diff for instance ids' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2758
- || ! grep -Fq 'no non-empty patches collected' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2759
- || ! grep -Fq 'collect-nan-instances' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2760
- || ! grep -Fq 'batch-nan-predictions' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2761
- || ! grep -Fq 'invalid JSON numeric constant: NaN' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2762
- || ! grep -Fq 'fetch-empty-limit' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2763
- || ! grep -Fq 'batch-empty-limit' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2764
- || ! grep -Fq 'no prediction instances selected' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2765
- || ! grep -Fq 'manifest malformed: expected JSON object' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2766
- || ! grep -Fq 'corpus-manifest-nan' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2767
- || ! grep -Fq 'loads_strict_json_object(manifest_path.read_text())' benchmark/auto-resolve/scripts/run-swebench-frozen-corpus.sh \
2768
- || ! grep -Fq 'loads_strict_json_object(pathlib.Path(sys.argv[1]).read_text())' benchmark/auto-resolve/scripts/run-swebench-frozen-corpus.sh \
2769
- || ! grep -Fq 'manifest malformed: prepared must be a non-empty array' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2770
- || ! grep -Fq 'prepared[1] expected JSON object' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2771
- || ! grep -Fq 'run ids malformed: no run ids' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2772
- || ! grep -Fq 'run ids malformed: line 2 is empty' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2773
- || ! grep -Fq 'run ids malformed: line 1 has unsafe run id' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2774
- || ! grep -Fq -- '--fixture must match [A-Za-z0-9_.-]+' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2775
- || ! grep -Fq -- '--run-id must match [A-Za-z0-9_.-]+' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2776
- || ! grep -Fq 'unsafe SWE-bench repo' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2777
- || ! grep -Fq 'unsafe SWE-bench base_commit' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2778
- || ! grep -Fq 'parse_constant=reject_json_constant' benchmark/auto-resolve/scripts/prepare-swebench-solver-worktree.py \
2779
- || ! grep -Fq 'parse_constant=reject_json_constant' benchmark/auto-resolve/scripts/fetch-swebench-instances.py \
2780
- || ! grep -Fq 'malformed fetched row 1: row must be object' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2781
- || ! grep -Fq 'solver-nan' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
2782
- || ! grep -Fq 'malformed scores_by_arm must not provide arm scores' benchmark/auto-resolve/scripts/test-iter-0033c-l1-summary.sh \
2783
- || ! grep -Fq 'non-dict result.json must not expose result fields' benchmark/auto-resolve/scripts/test-iter-0033c-l1-summary.sh \
2784
- || ! grep -Fq 'NaN result numeric fields must not appear in L1 summary' benchmark/auto-resolve/scripts/test-iter-0033c-l1-summary.sh \
2785
- || ! grep -Fq 'score_for(judge' benchmark/auto-resolve/scripts/iter-0033c-l1-summary.py \
2786
- || ! grep -Fq 'from pair_evidence_contract import is_score, is_strict_number' benchmark/auto-resolve/scripts/iter-0033c-l1-summary.py \
2787
- || ! grep -Fq 'loads_strict_json_object(path.read_text(encoding="utf8"))' benchmark/auto-resolve/scripts/iter-0033c-l1-summary.py \
2788
- || ! grep -Fq 'return legacy if is_score(legacy) else None' benchmark/auto-resolve/scripts/iter-0033c-l1-summary.py \
2789
- || ! grep -Fq 'def strict_number' benchmark/auto-resolve/scripts/iter-0033c-l1-summary.py \
2790
- || ! grep -Fq 'out-of-range scores must not appear in L1 summary' benchmark/auto-resolve/scripts/test-iter-0033c-l1-summary.sh \
2791
- || ! grep -Fq 'boolean scores must not appear in L1 summary' benchmark/auto-resolve/scripts/test-iter-0033c-l1-summary.sh \
2792
- || ! grep -Fq 'boolean result numeric fields must not appear in L1 summary' benchmark/auto-resolve/scripts/test-iter-0033c-l1-summary.sh \
2793
- || ! grep -Fq 'lowercase c_score' benchmark/auto-resolve/scripts/test-iter-0033c-l1-summary.sh; then
2794
- offenders="${offenders}"$'\n'"benchmark gates must require judge _blind_mapping before accepting score evidence"
2795
- fi
2796
- if ! grep -Fq 'solo_claude beats bare' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
2797
- || ! grep -Fq 'mapped_score' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
2798
- || ! grep -Fq 'parse_constant=reject_json_constant' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
2799
- || ! grep -Fq 'c1-nan-score' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
2800
- || ! grep -Fq 'dirty F5 L1<=L0 row must not be promoted' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
2801
- || ! grep -Fq 'l1-rerun-summary must not override pre-registered C1 selection grounds' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
2802
- || ! grep -Fq 'wrong mapping' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh; then
2803
- offenders="${offenders}"$'\n'"build-pair-eligible-manifest.py must use arm-mapped clean scores for F9 and L1<=L0 promotion"
2804
- fi
2805
- if [ -f benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F16-cli-quote-tax-rules/judge.json ] \
2806
- && ! python3 benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
2807
- --run-id 20260510-f16-f23-f25-combined-proof \
2808
- --pair-arm l2_risk_probes \
2809
- --min-fixtures 3 \
2810
- --min-pair-margin 5 \
2811
- --max-pair-solo-wall-ratio 3 \
2812
- --out-json /tmp/devlyn-lint-f16-f23-f25-pair-gate.json \
2813
- --out-md /tmp/devlyn-lint-f16-f23-f25-pair-gate.md >/dev/null 2>&1; then
2814
- offenders="${offenders}"$'\n'"current local artifacts for 20260510-f16-f23-f25-combined-proof must re-gate as PASS"
2815
- fi
2816
- if [ -f benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/F21-cli-scheduler-priority/judge.json ] \
2817
- && ! python3 benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
2818
- --run-id 20260511-f21-current-riskprobes-v1 \
2819
- --pair-arm l2_risk_probes \
2820
- --min-fixtures 1 \
2821
- --min-pair-margin 5 \
2822
- --max-pair-solo-wall-ratio 3 \
2823
- --out-json /tmp/devlyn-lint-f21-pair-gate.json \
2824
- --out-md /tmp/devlyn-lint-f21-pair-gate.md >/dev/null 2>&1; then
2825
- offenders="${offenders}"$'\n'"current local artifacts for 20260511-f21-current-riskprobes-v1 must re-gate as PASS"
2826
- fi
2827
- if ! grep -Fq 'bash scripts/lint-fixtures.sh' benchmark/auto-resolve/README.md; then
2828
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/README.md: gate-change instructions must include fixture schema lint"
2829
- fi
2830
- if ! python3 - <<'PY'
2831
- import runpy
2832
-
2833
- runtime = runpy.run_path("config/skills/_shared/verify-merge-findings.py")
2834
- benchmark = runpy.run_path("benchmark/auto-resolve/scripts/pair_evidence_contract.py")
2835
-
2836
- runtime_known = set(runtime["KNOWN_PAIR_TRIGGER_REASONS"])
2837
- canonical = set(benchmark["CANONICAL_PAIR_TRIGGER_REASONS"])
2838
- aliases = set(benchmark["HISTORICAL_PAIR_TRIGGER_REASON_ALIASES"])
2839
- normalized_aliases = set(benchmark["HISTORICAL_NORMALIZED_PAIR_TRIGGER_REASON_ALIASES"])
2840
- known = set(benchmark["KNOWN_PAIR_TRIGGER_REASONS"])
2841
- expected_aliases = {"risk_profile.high_risk", "risk_probes_enabled"}
2842
- expected_normalized_aliases = {
2843
- "complexity.high.spec.frontmatter",
2844
- "frontmatter.complexity.high",
2845
- "high.complexity.spec",
2846
- "high.risk.profile",
2847
- "spec.frontmatter.complexity.high",
2848
- "state.complexity.high",
2849
- }
2850
-
2851
- errors = []
2852
- if canonical != runtime_known:
2853
- errors.append("benchmark canonical pair-trigger reasons must match runtime reasons")
2854
- if aliases != expected_aliases:
2855
- errors.append("benchmark historical pair-trigger aliases changed unexpectedly")
2856
- if normalized_aliases != expected_normalized_aliases:
2857
- errors.append("benchmark normalized historical pair-trigger aliases changed unexpectedly")
2858
- if known != canonical | aliases:
2859
- errors.append("benchmark known pair-trigger reasons must be canonical plus aliases")
2860
- if runtime_known & aliases:
2861
- errors.append("runtime pair-trigger reasons must not accept benchmark-only aliases")
2862
- if not benchmark["has_historical_pair_trigger_reason"](["risk_profile.high_risk"]):
2863
- errors.append("benchmark historical alias helper must detect exact aliases")
2864
- if benchmark["has_historical_pair_trigger_reason"](["complexity.high"]):
2865
- errors.append("benchmark historical alias helper must not count canonical reasons")
2866
- if benchmark["is_known_pair_trigger_reason"]("risk high"):
2867
- errors.append("benchmark pair-trigger reader must reject normalized canonical reason strings")
2868
- if benchmark["is_canonical_pair_trigger_reason"]("risk high"):
2869
- errors.append("benchmark pair-trigger canonical helper must be exact-match only")
2870
- if not benchmark["is_known_pair_trigger_reason"]("high.risk.profile"):
2871
- errors.append("benchmark pair-trigger reader must preserve documented normalized historical aliases")
2872
- if benchmark["is_canonical_pair_trigger_reason"]("high.risk.profile"):
2873
- errors.append("benchmark normalized historical aliases must not count as canonical reasons")
2874
- if errors:
2875
- raise SystemExit("\n".join(errors))
2876
- PY
2877
- then
2878
- offenders="${offenders}"$'\n'"benchmark pair-trigger aliases must stay benchmark-only and runtime canonical reasons must stay in sync"
2879
- fi
2880
- if ! grep -Fq 'bash scripts/lint-shadow-fixtures.sh' benchmark/auto-resolve/README.md \
2881
- || ! grep -Fq 'The headroom and pair candidate runners' benchmark/auto-resolve/README.md \
2882
- || ! grep -Fq 'accept explicitly named `S*` ids for dry-run checks and candidate measurement' benchmark/auto-resolve/README.md \
2883
- || ! grep -Fq 'Use `run-suite.sh --suite shadow` only with `--dry-run`' benchmark/auto-resolve/README.md \
2884
- || ! grep -Fq 'rejected/smoke controls do not' benchmark/auto-resolve/README.md \
2885
- || ! grep -Fq 'npx devlyn-cli benchmark headroom --dry-run --min-fixtures 1 S1-cli-lang-flag' benchmark/auto-resolve/shadow-fixtures/README.md \
2886
- || ! grep -Fq 'Use non-dry-run headroom/pair only for' benchmark/auto-resolve/shadow-fixtures/README.md \
2887
- || ! grep -Fq 'explicitly named `S*` candidates with a solo-headroom hypothesis' benchmark/auto-resolve/shadow-fixtures/README.md \
2888
- || ! grep -Fq 'promote a validated `S*` task to an active `F*`' benchmark/auto-resolve/shadow-fixtures/README.md \
2889
- || ! grep -Fq 'shadow suite run-suite is dry-run only' benchmark/auto-resolve/scripts/run-suite.sh \
2890
- || ! grep -Fq 'run-suite.sh --suite shadow --dry-run' benchmark/auto-resolve/scripts/run-suite.sh \
2891
- || ! grep -Fq 'shadow suite refuses provider/judge runs' benchmark/auto-resolve/scripts/run-suite.sh \
2892
- || ! grep -Fq 'npx devlyn-cli benchmark suite --suite shadow --dry-run' bin/devlyn.js \
2893
- || ! grep -Fq 'Use benchmark headroom/pair with explicit S* candidates for real provider measurement.' benchmark/auto-resolve/scripts/run-suite.sh \
2894
- || ! grep -Fq 'shadow-suite-provider-run' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
2895
- || ! grep -Fq 'shadow-suite-judge-only-provider-run' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
2896
- || ! grep -Fq 'arg-parse-shadow-cli-suite-dry-run' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
2897
- || ! grep -Fq 'shadow suite dry-run must not invite a blocked non-dry-run suite invocation' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
2898
- || ! grep -Fq '`run-suite.sh --suite shadow` is dry-run only' benchmark/auto-resolve/shadow-fixtures/README.md \
2899
- || ! grep -Fq 'parallel to the active golden `F*` fixtures' benchmark/auto-resolve/shadow-fixtures/README.md \
2900
- || ! grep -Fq 'Active golden `F*` fixtures and pair-evidence audits control' benchmark/auto-resolve/shadow-fixtures/README.md \
2901
- || ! grep -Fq 'Default `--suite golden` keeps active golden `F*` behavior' benchmark/auto-resolve/shadow-fixtures/README.md \
2902
- || ! grep -Fq 'before any bare/solo/pair measurement' benchmark/auto-resolve/shadow-fixtures/README.md \
2903
- || ! grep -Fq 'Do not spend' benchmark/auto-resolve/shadow-fixtures/README.md \
2904
- || ! grep -Fq 'real provider calls on S1' benchmark/auto-resolve/shadow-fixtures/README.md \
2905
- || ! grep -Fq 'not a solo<pair evidence candidate' benchmark/auto-resolve/shadow-fixtures/README.md \
2906
- || grep -Fq 'F1-F9 still controls release' benchmark/auto-resolve/shadow-fixtures/README.md \
2907
- || grep -Fq 'existing F1-F9 behavior' benchmark/auto-resolve/shadow-fixtures/README.md \
2908
- || grep -Fq 'L0/L1/L2 measurement' benchmark/auto-resolve/shadow-fixtures/README.md; then
2909
- offenders="${offenders}"$'\n'"shadow fixture docs must describe lint and S* headroom/pair candidate calibration before golden promotion"
2910
- fi
2911
- if ! grep -Fq 'write a solo-headroom hypothesis' benchmark/auto-resolve/README.md \
2912
- || ! grep -Fq 'each new pair-candidate shadow fixture needs a' benchmark/auto-resolve/shadow-fixtures/README.md \
2913
- || ! grep -Fq "candidate's \`spec.md\`: name the visible behavior a capable \`solo_claude\`" benchmark/auto-resolve/README.md \
2914
- || ! grep -Fq '`solo_claude` baseline is expected to miss' benchmark/auto-resolve/shadow-fixtures/README.md \
2915
- || ! grep -Fq "candidate's \`spec.md\`" benchmark/auto-resolve/README.md \
2916
- || ! grep -Fq 'solo-headroom hypothesis in `spec.md`' benchmark/auto-resolve/shadow-fixtures/README.md \
2917
- || ! grep -Fq 'observable command from `expected.json`' benchmark/auto-resolve/README.md \
2918
- || ! grep -Fq 'observable command from' benchmark/auto-resolve/shadow-fixtures/README.md \
2919
- || ! grep -Fq 'candidate runners enforce this as an actionable hypothesis' benchmark/auto-resolve/README.md \
2920
- || ! grep -Fq 'runners enforce this as an actionable hypothesis' benchmark/auto-resolve/shadow-fixtures/README.md \
2921
- || ! grep -Fq 'backticked observable command matching `expected.json`' benchmark/auto-resolve/README.md \
2922
- || ! grep -Fq 'backticked observable command matching `expected.json`' benchmark/auto-resolve/shadow-fixtures/README.md \
2923
- || ! grep -Fq 'command/observable' benchmark/auto-resolve/README.md \
2924
- || ! grep -Fq 'command/observable' benchmark/auto-resolve/shadow-fixtures/README.md \
2925
- || ! grep -Fq 'itself containing `miss`' benchmark/auto-resolve/README.md \
2926
- || ! grep -Fq 'itself containing `miss`' benchmark/auto-resolve/shadow-fixtures/README.md \
2927
- || ! grep -Fq '## Solo ceiling avoidance' benchmark/auto-resolve/README.md \
2928
- || ! grep -Fq '## Solo ceiling avoidance' benchmark/auto-resolve/shadow-fixtures/README.md \
2929
- || ! grep -Fq 'solo-saturated `S2`-`S6` controls' benchmark/auto-resolve/README.md \
2930
- || ! grep -Fq 'calibrated solo-saturated controls (`S2`-`S6`)' benchmark/auto-resolve/shadow-fixtures/README.md; then
2931
- offenders="${offenders}"$'\n'"benchmark shadow candidate docs must require solo-headroom hypothesis and solo-ceiling avoidance before provider spend"
2932
- fi
2933
- if [ ! -f benchmark/auto-resolve/scripts/solo-headroom-hypothesis.py ] \
2934
- || ! grep -Fq 'def has_actionable_solo_headroom_hypothesis_text' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
2935
- || ! grep -Fq '"solo-headroom hypothesis" in lower' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
2936
- || ! grep -Fq 'and "solo_claude" in lower' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
2937
- || ! grep -Fq 'and "miss" in lower' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
2938
- || ! grep -Fq 'OBSERVABLE_COMMAND_MARKERS = ("command", "observable", "expose")' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
2939
- || ! grep -Fq 'def is_command_like_backtick' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
2940
- || ! grep -Fq 'def path_has_actionable_solo_headroom_hypothesis' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
2941
- || ! grep -Fq 'has_actionable_solo_headroom_hypothesis_text(text)' benchmark/auto-resolve/scripts/solo-headroom-hypothesis.py \
2942
- || ! grep -Fq 'actionable_observable_commands(text)' benchmark/auto-resolve/scripts/solo-headroom-hypothesis.py \
2943
- || ! grep -Fq -- '--expected-json' benchmark/auto-resolve/scripts/solo-headroom-hypothesis.py \
2944
- || ! grep -Fq 'expected_commands(args.expected_json)' benchmark/auto-resolve/scripts/solo-headroom-hypothesis.py \
2945
- || ! grep -Fq 'expected UTF-8 text' benchmark/auto-resolve/scripts/solo-headroom-hypothesis.py \
2946
- || ! grep -Fq 'solo-headroom-hypothesis.py' scripts/lint-shadow-fixtures.sh \
2947
- || ! grep -Fq 'has_solo_ceiling_avoidance_note' scripts/lint-shadow-fixtures.sh \
2948
- || ! grep -Fq 'solo-ceiling-avoidance.py' scripts/lint-shadow-fixtures.sh \
2949
- || ! grep -Fq 'SECTION_RE' benchmark/auto-resolve/scripts/solo-ceiling-avoidance.py \
2950
- || ! grep -Fq 'CONTROL_RE' benchmark/auto-resolve/scripts/solo-ceiling-avoidance.py \
2951
- || ! grep -Fq 'REASON_RE' benchmark/auto-resolve/scripts/solo-ceiling-avoidance.py \
2952
- || ! grep -Fq 'expected UTF-8 text' benchmark/auto-resolve/scripts/solo-ceiling-avoidance.py \
2953
- || ! grep -Fq 'fixture_has_solo_ceiling_avoidance_note' benchmark/auto-resolve/scripts/run-headroom-candidate.sh \
2954
- || ! grep -Fq 'fixture_has_solo_ceiling_avoidance_note' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
2955
- || ! grep -Fq 'solo-ceiling-avoidance.py' benchmark/auto-resolve/scripts/run-headroom-candidate.sh \
2956
- || ! grep -Fq 'solo-ceiling-avoidance.py' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
2957
- || ! grep -Fq 'shadow fixture NOTES.md needs ## Solo ceiling avoidance' benchmark/auto-resolve/scripts/run-headroom-candidate.sh \
2958
- || ! grep -Fq 'shadow fixture NOTES.md needs ## Solo ceiling avoidance' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
2959
- || ! grep -Fq 'solo-headroom-hypothesis.py' benchmark/auto-resolve/scripts/run-headroom-candidate.sh \
2960
- || ! grep -Fq 'solo-headroom-hypothesis.py' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
2961
- || ! grep -Fq -- '--expected-json "$dir/expected.json" "$dir/spec.md"' benchmark/auto-resolve/scripts/run-headroom-candidate.sh \
2962
- || ! grep -Fq -- '--expected-json "$dir/expected.json" "$dir/spec.md"' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
2963
- || ! grep -Fq -- '--expected-json "$d/expected.json" "$spec"' scripts/lint-shadow-fixtures.sh \
2964
- || ! grep -Fq 'from expected.json before provider spend' benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh \
2965
- || ! grep -Fq 'from expected.json before provider spend' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
2966
- || ! grep -Fq 'unrelated-backtick-solo-headroom-hypothesis' benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh \
2967
- || ! grep -Fq 'unrelated-backtick-solo-headroom-hypothesis' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
2968
- || ! grep -Fq 'observable-without-miss-solo-headroom-hypothesis' benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh \
2969
- || ! grep -Fq 'observable-without-miss-solo-headroom-hypothesis' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
2970
- || ! grep -Fq 'missing-solo-ceiling-avoidance' benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh \
2971
- || ! grep -Fq 'missing-solo-ceiling-avoidance' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
2972
- || ! grep -Fq 'weak-solo-ceiling-avoidance' benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh \
2973
- || ! grep -Fq 'weak-solo-ceiling-avoidance' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
2974
- || ! grep -Fq 'hypothesis with unrelated backtick must fail' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
2975
- || ! grep -Fq 'weak hypothesis without observable command must fail' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
2976
- || ! grep -Fq 'actionable-hypothesis.md' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
2977
- || ! grep -Fq 'docs-style-actionable-hypothesis.md' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
2978
- || ! grep -Fq 'shadow-missing-solo-ceiling-avoidance' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
2979
- || ! grep -Fq 'shadow-weak-solo-ceiling-avoidance' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
2980
- || ! grep -Fq 'weak solo ceiling avoidance must fail' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
2981
- || ! grep -Fq 'non-utf8-solo-ceiling.md' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
2982
- || ! grep -Fq 'non-utf8-hypothesis.md' benchmark/auto-resolve/scripts/test-lint-fixtures.sh; then
2983
- offenders="${offenders}"$'\n'"solo-headroom hypothesis provider-spend guards must share one actionable checker and test weak-vs-actionable cases"
2984
- fi
2985
- if ! grep -Fq 'pair-candidate-frontier.py' benchmark/auto-resolve/README.md \
2986
- || ! grep -Fq 'npx devlyn-cli benchmark frontier --out-md /tmp/devlyn-pair-frontier.md' benchmark/auto-resolve/README.md \
2987
- || ! grep -Fq 'npx devlyn-cli benchmark frontier --out-md /tmp/devlyn-pair-frontier.md' benchmark/auto-resolve/run-real-benchmark.md \
2988
- || ! grep -Fq 'npx devlyn-cli benchmark audit --out-dir /tmp/devlyn-benchmark-audit' README.md \
2989
- || ! grep -Fq 'npx devlyn-cli benchmark audit --out-dir /tmp/devlyn-benchmark-audit' benchmark/auto-resolve/README.md \
2990
- || ! grep -Fq 'npx devlyn-cli benchmark audit --out-dir /tmp/devlyn-benchmark-audit' benchmark/auto-resolve/run-real-benchmark.md \
2991
- || ! grep -Fq 'npx devlyn-cli benchmark audit --require-hypothesis-trigger --out-dir /tmp/devlyn-benchmark-audit-strict' README.md \
2992
- || ! grep -Fq 'npx devlyn-cli benchmark audit --require-hypothesis-trigger --out-dir /tmp/devlyn-benchmark-audit-strict' benchmark/auto-resolve/README.md \
2993
- || ! grep -Fq 'npx devlyn-cli benchmark audit --require-hypothesis-trigger --out-dir /tmp/devlyn-benchmark-audit-strict' benchmark/auto-resolve/run-real-benchmark.md \
2994
- || ! grep -Fq 'npx devlyn-cli benchmark audit --require-hypothesis-trigger --out-dir /tmp/devlyn-benchmark-audit-strict' bin/devlyn.js \
2995
- || ! grep -Fq 'npx devlyn-cli benchmark audit --require-hypothesis-trigger --out-dir /tmp/devlyn-benchmark-audit-strict' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
2996
- || ! grep -Fq 'npx devlyn-cli benchmark frontier Show pair candidate frontier scores/triggers without providers' bin/devlyn.js \
2997
- || ! grep -Fq 'npx devlyn-cli benchmark frontier Show pair candidate frontier scores/triggers without providers' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
2998
- || ! grep -Fq 'npx devlyn-cli benchmark audit Audit pair evidence readiness' bin/devlyn.js \
2999
- || ! grep -Fq 'npx devlyn-cli benchmark audit Audit pair evidence readiness' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3000
- || ! grep -Fq 'Show active rejected/evidence/unmeasured pair candidates, scores, and triggers without providers' bin/devlyn.js \
3001
- || ! grep -Fq 'Show active rejected/evidence/unmeasured pair candidates, scores, and triggers without providers' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3002
- || ! grep -Fq 'Prints pair evidence score rows with trigger reasons; --out-md includes a Triggers column' bin/devlyn.js \
3003
- || ! grep -Fq 'Prints pair evidence score rows with trigger reasons; --out-md includes a Triggers column' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3004
- || ! grep -Fq 'Prints frontier score rows plus headroom and pair quality handoff rows' bin/devlyn.js \
3005
- || ! grep -Fq 'Prints frontier score rows plus headroom and pair quality handoff rows' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3006
- || ! grep -Fq 'Prints frontier score rows plus headroom_rejections=PASS/FAIL, pair_evidence_quality=PASS/FAIL, pair_trigger_reasons=PASS/FAIL, pair_evidence_hypotheses=PASS/FAIL, pair_evidence_hypothesis_triggers=PASS/WARN/FAIL, historical-alias, and hypothesis-trigger gap handoff rows' bin/devlyn.js \
3007
- || ! grep -Fq 'Prints frontier score rows plus headroom_rejections=PASS/FAIL, pair_evidence_quality=PASS/FAIL, pair_trigger_reasons=PASS/FAIL, pair_evidence_hypotheses=PASS/FAIL, pair_evidence_hypothesis_triggers=PASS/WARN/FAIL, historical-alias, and hypothesis-trigger gap handoff rows' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3008
- || ! grep -Fq "audit: 'audit-pair-evidence.py'" bin/devlyn.js \
3009
- || ! grep -Fq 'npx devlyn-cli benchmark audit [options]' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3010
- || ! grep -Fq -- '--min-pair-evidence N default: 4' bin/devlyn.js \
3011
- || ! grep -Fq -- '--min-pair-evidence N default: 4' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3012
- || ! grep -Fq -- '--min-pair-margin N default: 5' bin/devlyn.js \
3013
- || ! grep -Fq -- '--min-pair-margin N default: 5' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3014
- || ! grep -Fq -- '--max-pair-solo-wall-ratio N default: 3' bin/devlyn.js \
3015
- || ! grep -Fq -- '--max-pair-solo-wall-ratio N default: 3' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3016
- || ! grep -Fq -- '--require-hypothesis-trigger' bin/devlyn.js \
3017
- || ! grep -Fq -- '--require-hypothesis-trigger' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3018
- || ! grep -Fq -- '--require-hypothesis-trigger' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3019
- || ! grep -Fq 'npx devlyn-cli benchmark audit-headroom --out-json /tmp/devlyn-headroom-audit.json' README.md \
3020
- || ! grep -Fq 'npx devlyn-cli benchmark audit-headroom --out-json /tmp/devlyn-headroom-audit.json' benchmark/auto-resolve/README.md \
3021
- || ! grep -Fq 'npx devlyn-cli benchmark audit-headroom --out-json /tmp/devlyn-headroom-audit.json' benchmark/auto-resolve/run-real-benchmark.md \
3022
- || ! grep -Fq 'npx devlyn-cli benchmark audit-headroom Audit failed headroom results' bin/devlyn.js \
3023
- || ! grep -Fq 'npx devlyn-cli benchmark audit-headroom Audit failed headroom results' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3024
- || ! grep -Fq "'audit-headroom': 'audit-headroom-rejections.py'" bin/devlyn.js \
3025
- || ! grep -Fq 'npx devlyn-cli benchmark audit-headroom [options]' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3026
- || ! grep -Fq 'benchmarkMode === '\''frontier'\''' bin/devlyn.js \
3027
- || ! grep -Fq 'test-pair-candidate-frontier.sh' benchmark/auto-resolve/README.md \
3028
- || ! grep -Fq 'audit-pair-evidence.py' benchmark/auto-resolve/README.md \
3029
- || ! grep -Fq 'PASS audit-pair-evidence' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3030
- || ! grep -Fq 'FAIL audit-pair-evidence' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3031
- || ! grep -Fq 'FAIL audit-pair-evidence' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3032
- || ! grep -Fq 'audit.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3033
- || ! grep -Fq 'audit.json' README.md \
3034
- || ! grep -Fq 'audit.json' benchmark/auto-resolve/README.md \
3035
- || ! grep -Fq 'audit.json' benchmark/auto-resolve/run-real-benchmark.md \
3036
- || ! grep -Fq '"frontier_summary": frontier_summary' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3037
- || ! grep -Fq '"pair_evidence_rows": pair_evidence_rows' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3038
- || ! grep -Fq 'def load_pair_evidence_rows' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3039
- || ! grep -Fq 'def load_frontier_stdout_metrics' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3040
- || ! grep -Fq '"summary_rows": summary_rows' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3041
- || ! grep -Fq '"aggregate_rows": aggregate_rows' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3042
- || ! grep -Fq '"trigger_rows": trigger_rows' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3043
- || ! grep -Fq '"trigger_rows_match_count": trigger_rows == expected_rows' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3044
- || ! grep -Fq '"hypothesis_trigger_rows_match_count": hypothesis_trigger_rows == expected_rows' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3045
- || ! grep -Fq 'from pair_evidence_contract import' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3046
- || ! grep -Fq 'from pair_evidence_contract import' benchmark/auto-resolve/scripts/audit-headroom-rejections.py \
3047
- || ! grep -Fq 'from pair_evidence_contract import' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
3048
- || ! grep -Fq 'def reject_json_constant' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3049
- || ! grep -Fq 'def loads_strict_json_object' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3050
- || ! grep -Fq 'parse_constant=reject_json_constant' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3051
- || ! grep -Fq 'loads_strict_json_object(path.read_text())' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
3052
- || ! grep -Fq 'loads_strict_json_object(path.read_text())' benchmark/auto-resolve/scripts/headroom-gate.py \
3053
- || ! grep -Fq 'loads_strict_json_object(path.read_text())' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
3054
- || ! grep -Fq 'loads_strict_json_object(path.read_text())' benchmark/auto-resolve/scripts/audit-headroom-rejections.py \
3055
- || ! grep -Fq 'loads_strict_json_object(path.read_text())' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3056
- || ! grep -Fq 'nan-json-constant' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
3057
- || ! grep -Fq 'nan-result-json' benchmark/auto-resolve/scripts/test-headroom-gate.sh \
3058
- || ! grep -Fq 'nan-result-json' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
3059
- || ! grep -Fq 'CANONICAL_PAIR_TRIGGER_REASONS = {' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3060
- || ! grep -Fq 'HISTORICAL_PAIR_TRIGGER_REASON_ALIASES = {' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3061
- || ! grep -Fq 'HISTORICAL_NORMALIZED_PAIR_TRIGGER_REASON_ALIASES = {' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3062
- || ! grep -Fq 'Benchmark readers accept historical aliases only for archived artifacts' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3063
- || ! grep -Fq 'KNOWN_PAIR_TRIGGER_REASONS = (' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3064
- || ! grep -Fq 'mode.pair-verify' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3065
- || ! grep -Fq 'def normalized_pair_trigger_reason' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3066
- || ! grep -Fq 'def is_canonical_pair_trigger_reason' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3067
- || ! grep -Fq 'def has_known_pair_trigger_reason' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3068
- || ! grep -Fq 'def all_known_pair_trigger_reasons' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3069
- || ! grep -Fq 'def has_canonical_pair_trigger_reason' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3070
- || ! grep -Fq 'spec.solo_headroom_hypothesis' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3071
- || ! grep -Fq 'risk_profile.high_risk' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3072
- || ! grep -Fq 'has_known_pair_trigger_reason(reasons)' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
3073
- || ! grep -Fq 'all_known_pair_trigger_reasons(reasons)' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
3074
- || ! grep -Fq 'has_canonical_pair_trigger_reason(reasons)' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
3075
- || ! grep -Fq '"pair_trigger_has_canonical_reason": has_canonical_pair_trigger_reason(' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
3076
- || ! grep -Fq '"pair_trigger_has_hypothesis_reason": (' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
3077
- || ! grep -Fq '"require_hypothesis_trigger": args.require_hypothesis_trigger' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
3078
- || ! grep -Fq '"pair_trigger_has_canonical_reason": true' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
3079
- || ! grep -Fq '"pair_trigger_has_hypothesis_reason": true' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
3080
- || ! grep -Fq '"require_hypothesis_trigger": true' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
3081
- || ! grep -Fq 'pair_trigger eligible with a canonical reason' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
3082
- || ! grep -Fq 'pair_trigger eligible with canonical reason' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
3083
- || ! grep -Fq 'pair_trigger reasons missing known trigger reason' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
3084
- || ! grep -Fq 'pair_trigger reasons contain unknown trigger reason' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
3085
- || ! grep -Fq 'pair_trigger reasons missing canonical trigger reason' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
3086
- || ! grep -Fq 'fixture_spec_has_solo_headroom_hypothesis(fixture_dir.name)' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
3087
- || ! grep -Fq 'pair_trigger missing spec.solo_headroom_hypothesis' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
3088
- || ! grep -Fq -- '--require-hypothesis-trigger' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
3089
- || ! grep -Fq -- '--require-hypothesis-trigger' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
3090
- || ! grep -Fq 'release audit: npx devlyn-cli benchmark audit --require-hypothesis-trigger --out-dir /tmp/devlyn-benchmark-audit-strict' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
3091
- || ! grep -Fq 'release audit: npx devlyn-cli benchmark audit --require-hypothesis-trigger --out-dir /tmp/devlyn-benchmark-audit-strict' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
3092
- || ! grep -Fq 'missing-hypothesis-trigger' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
3093
- || ! grep -Fq 'grep -Fq -- '\''--require-hypothesis-trigger'\''' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
3094
- || ! grep -Fq 'def fmt_trigger_reasons' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
3095
- || ! grep -Fq 'Hypothesis trigger required: {str(report' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
3096
- || ! grep -Fq '| Fixture | Bare | Bare headroom | Solo_claude | Solo_claude headroom | Pair | Margin | Pair mode | Hypothesis trigger | Triggers | Wall ratio | Status | Reason |' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
3097
- || ! grep -Fq '| Fixture | Bare | Bare headroom | Solo_claude | Solo_claude headroom | Pair | Margin | Pair mode | Hypothesis trigger | Triggers | Wall ratio | Status | Reason |' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
3098
- || ! grep -Fq 'trigger-reason, and wall-ratio columns' benchmark/auto-resolve/README.md \
3099
- || ! grep -Fq 'eligible with non-empty reasons and at least one canonical reason' benchmark/auto-resolve/README.md \
3100
- || ! grep -Fq 'fixtures with an actionable solo-headroom hypothesis must include `spec.solo_headroom_hypothesis` in the trigger reasons' benchmark/auto-resolve/README.md \
3101
- || ! grep -Fq 'pair trigger eligibility, trigger reasons, canonical-trigger coverage, and `spec.solo_headroom_hypothesis` coverage' benchmark/auto-resolve/run-real-benchmark.md \
3102
- || ! grep -Fq 'unknown-pair-trigger-reason' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
3103
- || ! grep -Fq 'mixed-unknown-pair-trigger-reason' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
3104
- || ! grep -Fq 'normalized-canonical-pair-trigger-reason' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
3105
- || ! grep -Fq 'historical-only-pair-trigger-reason' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
3106
- || ! grep -Fq 'unknown-pair-trigger' benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh \
3107
- || ! grep -Fq 'def normalize_pair_evidence_row' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3108
- || ! grep -Fq 'def best_pair_evidence' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3109
- || ! grep -Fq 'ALLOWED_PAIR_ARMS = {"l2_risk_probes", "l2_gated"}' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3110
- || ! grep -Fq 'pair_arm not in ALLOWED_PAIR_ARMS' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3111
- || ! grep -Fq 'pair_mode = row.get("pair_mode")' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3112
- || ! grep -Fq 'if pair_mode is not True:' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3113
- || ! grep -Fq 'pair_trigger_eligible = row.get("pair_trigger_eligible")' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3114
- || ! grep -Fq 'if pair_trigger_eligible is not True:' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3115
- || ! grep -Fq 'pair_trigger_reasons = row.get("pair_trigger_reasons")' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3116
- || ! grep -Fq 'isinstance(pair_trigger_reasons, list)' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3117
- || ! grep -Fq 'all_known_pair_trigger_reasons(pair_trigger_reasons)' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3118
- || ! grep -Fq 'has_canonical_pair_trigger_reason(pair_trigger_reasons)' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3119
- || ! grep -Fq '"pair_trigger_reasons": pair_trigger_reasons' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3120
- || ! grep -Fq '"pair_trigger_has_canonical_reason": True' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3121
- || ! grep -Fq '"pair_trigger_has_hypothesis_reason": (' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3122
- || ! grep -Fq 'def pair_result_trigger_reasons' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
3123
- || ! grep -Fq 'candidate_row = dict(row)' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
3124
- || ! grep -Fq 'all_known_pair_trigger_reasons(reasons)' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
3125
- || ! grep -Fq 'has_canonical_pair_trigger_reason(reasons)' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
3126
- || ! grep -Fq 'candidate_row["pair_trigger_reasons"] = reasons' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
3127
- || ! grep -Fq '"reasons": ["complexity.high", "looks-hard"]' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
3128
- || ! grep -Fq '"reasons":["risk high"]' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
3129
- || ! grep -Fq 'triggers={triggers}' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
3130
- || ! grep -Fq 'hypothesis_trigger={hypothesis_trigger}' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
3131
- || ! grep -Fq '| Fixture | Status | Verdict | Evidence | Pair arm | Triggers | Hypothesis trigger |' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
3132
- || ! grep -Fq 'pair-arm, and trigger-reason columns' benchmark/auto-resolve/README.md \
3133
- || ! grep -Fq '| Fixture | Status | Verdict | Evidence | Pair arm | Triggers | Hypothesis trigger |' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
3134
- || ! grep -Fq 'def check_pair_evidence_hypotheses' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3135
- || ! grep -Fq 'def pair_evidence_hypothesis_trigger_rows' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3136
- || ! grep -Fq 'def pair_evidence_hypothesis_trigger_gap_details' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3137
- || ! grep -Fq 'def check_pair_evidence_hypothesis_triggers' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3138
- || ! grep -Fq 'def print_pair_evidence_hypothesis_triggers_summary' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3139
- || ! grep -Fq 'pair_evidence_hypotheses={status}' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3140
- || ! grep -Fq 'pair_evidence_hypothesis_triggers={status}' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3141
- || ! grep -Fq 'pair evidence hypotheses missing for fixture(s)' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3142
- || ! grep -Fq 'pair-evidence-hypotheses' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3143
- || ! grep -Fq 'pair_evidence_hypotheses=PASS documented=2 total=2' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3144
- || ! grep -Fq 'pair_evidence_hypothesis_triggers=WARN matched=0 documented=2 total=2' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3145
- || ! grep -Fq 'pair_evidence_hypothesis_trigger_gaps=F16-cli-quote-tax-rules=complexity.high;F21-cli-scheduler-priority=complexity.high,risk_profile.high_risk' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3146
- || ! grep -Fq 'pair_evidence_hypothesis_triggers=FAIL matched=0 documented=2 total=2' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3147
- || ! grep -Fq 'strict benchmark audit must not report current hypothesis-trigger gaps' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3148
- || ! grep -Fq 'pair_evidence_hypothesis_triggers=PASS matched=4 documented=4 total=4' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3149
- || ! grep -Fq 'verdict=pair_evidence_passed triggers={triggers}' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3150
- || ! grep -Fq 'def format_trigger_reasons' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3151
- || ! grep -Fq 'missing-frontier-triggers' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3152
- || ! grep -Fq 'triggers=complexity.high' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
3153
- || ! grep -Fq '"pair_trigger_reasons": pair_trigger_reasons(pair_result)' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
3154
- || ! grep -Fq '"pair_trigger_eligible": pair_trigger_eligible' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3155
- || ! grep -Fq '"verdict": "pair_evidence_passed"' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3156
- || ! grep -Fq '"pair_mode": pair_mode' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3157
- || ! grep -Fq 'def pair_result_trigger_reasons' benchmark/auto-resolve/scripts/audit-headroom-rejections.py \
3158
- || ! grep -Fq 'candidate_row = dict(row)' benchmark/auto-resolve/scripts/audit-headroom-rejections.py \
3159
- || ! grep -Fq 'candidate_row["pair_trigger_reasons"] = reasons' benchmark/auto-resolve/scripts/audit-headroom-rejections.py \
3160
- || ! grep -Fq 'F16-cli-quote-tax-rules/l2_risk_probes/result.json' benchmark/auto-resolve/scripts/test-audit-headroom-rejections.sh \
3161
- || ! grep -Fq '"reasons":["risk high"]' benchmark/auto-resolve/scripts/test-audit-headroom-rejections.sh \
3162
- || ! grep -Fq 'def is_strict_int' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3163
- || ! grep -Fq 'def is_score' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3164
- || ! grep -Fq '0 <= value <= 100' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3165
- || ! grep -Fq 'def is_strict_number' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3166
- || ! grep -Fq 'import math' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3167
- || ! grep -Fq 'math.isfinite(value)' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3168
- || ! grep -Fq 'and value > 0' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3169
- || ! grep -Fq 'if pair_margin != pair_score - solo_score:' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3170
- || ! grep -Fq '"min_pair_evidence": min_pair_evidence' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3171
- || ! grep -Fq '"actual_rows": len(pair_evidence_rows)' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3172
- || ! grep -Fq '"rows_match_count": (' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3173
- || ! grep -Fq '"pair_evidence_quality": {' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3174
- || ! grep -Fq 'def check_pair_evidence_quality' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3175
- || ! grep -Fq 'def print_pair_evidence_quality' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3176
- || ! grep -Fq 'def check_pair_trigger_reasons' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3177
- || ! grep -Fq 'def print_pair_trigger_reasons_summary' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3178
- || ! grep -Fq 'def pair_trigger_historical_alias_details' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3179
- || ! grep -Fq 'pair_evidence_quality={status} min_pair_margin_actual={actual_margin}' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3180
- || ! grep -Fq 'pair_trigger_reasons={status} canonical={canonical} historical_alias={historical_alias}' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3181
- || ! grep -Fq '"historical_alias_details": historical_alias_details' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3182
- || ! grep -Fq 'frontier pair_margin_min does not match pair evidence rows' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3183
- || ! grep -Fq 'frontier pair_solo_wall_ratio_max does not match pair evidence rows' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3184
- || ! grep -Fq 'is_strict_int(pair_evidence_count)' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3185
- || ! grep -Fq 'if not is_strict_int(count):' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3186
- || ! grep -Fq 'pair evidence count missing or malformed from frontier report' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3187
- || ! grep -Fq 'pair evidence rows {len(rows)} do not match summary count {count}' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3188
- || ! grep -Fq 'def check_frontier_report' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3189
- || ! grep -Fq 'frontier verdict {verdict!r} is not PASS' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3190
- || ! grep -Fq 'frontier has {unmeasured_count} unmeasured candidate fixture(s)' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3191
- || ! grep -Fq '"frontier_report": {' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3192
- || ! grep -Fq '"min_pair_margin": min_pair_margin' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3193
- || ! grep -Fq '"max_pair_solo_wall_ratio": max_pair_solo_wall_ratio' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3194
- || ! grep -Fq 'not math.isfinite(args.max_pair_solo_wall_ratio)' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3195
- || ! grep -Fq 'max-pair-solo-wall-ratio must be finite and > 0' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3196
- || ! grep -Fq 'pair evidence count {count} below required minimum {minimum}' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3197
- || ! grep -Fq 'assert audit["checks"]["min_pair_evidence"]["actual_rows"] == len(audit["pair_evidence_rows"])' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3198
- || ! grep -Fq 'assert audit["checks"]["frontier_report"]["verdict"] == frontier["verdict"]' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3199
- || ! grep -Fq 'pair_evidence_quality=PASS min_pair_margin_actual=+21 min_pair_margin_required=+5 max_wall_actual=2.25x max_wall_allowed=3.00x' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3200
- || ! grep -Fq 'pair_trigger_reasons=PASS canonical=4 historical_alias=0 exposed=4 total=4 summary=4 rows_match=true' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3201
- || ! grep -Fq 'current benchmark audit must not report historical aliases or hypothesis-trigger gaps' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3202
- || ! grep -Fq 'benchmark audit must fail when min pair evidence exceeds current evidence rows' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3203
- || ! grep -Fq 'actual_pair_evidence=$(python3 - "$TMP/audit/audit.json"' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3204
- || ! grep -Fq 'required_pair_evidence=$((actual_pair_evidence + 1))' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3205
- || ! grep -Fq 'pair evidence count ${actual_pair_evidence} below required minimum ${required_pair_evidence}' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3206
- || ! grep -Fq 'pair_margin_avg=+27.25 pair_margin_min=+21 wall_avg=1.66x wall_max=2.25x' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3207
- || ! grep -Fq 'F16-cli-quote-tax-rules: bare=50 solo_claude=75 pair=96 arm=l2_risk_probes margin=+21' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3208
- || ! grep -Fq 'FAIL audit-pair-evidence' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3209
- || ! grep -Fq 'assert audit["checks"]["min_pair_evidence"]["required"] == required' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3210
- || ! grep -Fq 'assert audit["checks"]["pair_evidence_quality"]["status"] == "PASS"' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3211
- || ! grep -Fq 'assert audit["checks"]["pair_trigger_reasons"]["summary_pair_evidence_count"] == 4' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3212
- || ! grep -Fq 'assert audit["checks"]["pair_trigger_reasons"]["rows_match_count"] is True' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3213
- || ! grep -Fq 'assert audit["checks"]["pair_trigger_reasons"]["canonical_rows"] == 4' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3214
- || ! grep -Fq 'assert audit["checks"]["pair_trigger_reasons"]["historical_alias_details"] == []' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3215
- || ! grep -Fq 'assert len(audit["pair_evidence_rows"]) == frontier["pair_evidence_count"]' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3216
- || ! grep -Fq 'benchmark frontier must fail when active unmeasured candidates remain' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3217
- || ! grep -Fq 'PASS pair-candidate-frontier' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3218
- || ! grep -Fq 'FAIL pair-candidate-frontier' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3219
- || ! grep -Fq 'assert report["rows"][0]["status"] == "candidate_unmeasured"' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3220
- || ! grep -Fq 'assert row["verdict"] == "pair_evidence_passed"' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3221
- || ! grep -Fq 'assert row["pair_mode"] is True' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3222
- || ! grep -Fq 'assert row["pair_trigger_eligible"] is True' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3223
- || ! grep -Fq 'assert isinstance(row["pair_solo_wall_ratio"], (int, float))' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3224
- || ! grep -Fq 'assert report["frontier_summary"]["pair_margin_avg"] == 27' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3225
- || ! grep -Fq 'assert report["pair_evidence_rows"] == [' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3226
- || ! grep -Fq 'assert report["checks"]["min_pair_evidence"]["actual_rows"] == 2' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3227
- || ! grep -Fq 'assert report["checks"]["min_pair_evidence"]["rows_match_count"] is True' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3228
- || ! grep -Fq 'assert report["checks"]["min_pair_evidence"]["status"] == "FAIL"' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3229
- || ! grep -Fq 'assert report["checks"]["pair_evidence_quality"]["status"] == "PASS"' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3230
- || ! grep -Fq 'assert report["checks"]["pair_trigger_reasons"]["status"] == "PASS"' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3231
- || ! grep -Fq 'assert report["checks"]["pair_trigger_reasons"]["summary_pair_evidence_count"] == 2' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3232
- || ! grep -Fq 'assert report["checks"]["pair_trigger_reasons"]["rows_match_count"] is True' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3233
- || ! grep -Fq 'assert report["checks"]["pair_trigger_reasons"]["historical_alias_details"] == [' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3234
- || ! grep -Fq 'normalized-canonical-trigger-reason-rows' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3235
- || ! grep -Fq 'assert report["checks"]["pair_evidence_quality"]["min_pair_margin_actual"] == 21' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3236
- || ! grep -Fq 'pair_evidence_quality=PASS min_pair_margin_actual=+21 min_pair_margin_required=+5 max_wall_actual=1.47x max_wall_allowed=3.00x' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3237
- || ! grep -Fq 'pair_evidence_quality=FAIL min_pair_margin_actual=+4 min_pair_margin_required=+5 max_wall_actual=1.20x max_wall_allowed=3.00x' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3238
- || ! grep -Fq 'no-quality-rows-frontier.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3239
- || ! grep -Fq 'pair evidence quality check has no complete rows' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3240
- || ! grep -Fq 'low-quality-frontier.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3241
- || ! grep -Fq 'pair evidence margin below minimum for fixture(s): F16-cli-quote-tax-rules' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3242
- || ! grep -Fq 'high-wall-frontier.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3243
- || ! grep -Fq 'pair evidence wall ratio above maximum for fixture(s): F16-cli-quote-tax-rules' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3244
- || ! grep -Fq 'summary-mismatch-frontier.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3245
- || ! grep -Fq 'frontier pair_margin_min does not match pair evidence rows' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3246
- || ! grep -Fq 'summary-wall-mismatch-frontier.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3247
- || ! grep -Fq 'frontier pair_solo_wall_ratio_max does not match pair evidence rows' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3248
- || ! grep -Fq 'frontier-fail-verdict.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3249
- || ! grep -Fq 'frontier-unmeasured.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3250
- || ! grep -Fq 'frontier-malformed-unmeasured.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3251
- || ! grep -Fq 'frontier-incomplete-best.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3252
- || ! grep -Fq '"run_id": "lower-complete"' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3253
- || ! grep -Fq 'bool-frontier-count.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3254
- || ! grep -Fq 'malformed-pair-evidence-count' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3255
- || ! grep -Fq 'mismatched-frontier-rows.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3256
- || ! grep -Fq 'pair evidence rows 1 do not match summary count 2' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3257
- || ! grep -Fq 'bad-frontier-row-fields.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3258
- || ! grep -Fq 'nan-frontier-row-fields.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3259
- || ! grep -Fq 'nan-max-wall-ratio' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3260
- || ! grep -Fq 'mismatched-margin-row-fields.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3261
- || ! grep -Fq 'mismatched-margin-row-fields' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3262
- || ! grep -Fq 'overrange-score-row-fields.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3263
- || ! grep -Fq 'overrange-score-row-fields' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3264
- || ! grep -Fq 'invalid-pair-arm-row-fields.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3265
- || ! grep -Fq 'invalid-pair-arm-row-fields' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3266
- || ! grep -Fq 'false-pair-mode-row-fields.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3267
- || ! grep -Fq 'false-pair-mode-row-fields' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3268
- || ! grep -Fq 'zero-wall-row-fields.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3269
- || ! grep -Fq 'zero-wall-row-fields' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3270
- || ! grep -Fq 'pair evidence count 2 below required minimum 4' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3271
- || ! grep -Fq 'assert rows["F22-cli-low-margin"]["status"] == "candidate_unmeasured"' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
3272
- || ! grep -Fq 'assert rows["F22-cli-low-margin"]["status"] == "pair_evidence_passed"' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
3273
- || ! grep -Fq 'assert rows["F23-cli-high-wall"]["status"] == "candidate_unmeasured"' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
3274
- || ! grep -Fq 'assert rows["F23-cli-high-wall"]["status"] == "pair_evidence_passed"' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
3275
- || ! grep -Fq 'def print_final_verdict' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
3276
- || ! grep -Fq 'PASS pair-candidate-frontier' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
3277
- || ! grep -Fq 'FAIL pair-candidate-frontier' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
3278
- || ! grep -Fq 'PASS pair-candidate-frontier' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
3279
- || ! grep -Fq 'FAIL pair-candidate-frontier' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
3280
- || ! grep -Fq 'requires at least four fixtures with passing pair evidence' README.md \
3281
- || ! grep -Fq 'revalidates frontier `verdict: PASS`, zero unmeasured candidates' README.md \
3282
- || ! grep -Fq 'requires at least four active fixtures with passing pair evidence' benchmark/auto-resolve/README.md \
3283
- || ! grep -Fq 'revalidates frontier `verdict: PASS`' benchmark/auto-resolve/README.md \
3284
- || ! grep -Fq 'requires at least four active fixtures with passing pair evidence' benchmark/auto-resolve/run-real-benchmark.md \
3285
- || ! grep -Fq 'revalidates frontier `verdict: PASS`, zero unmeasured candidates' benchmark/auto-resolve/run-real-benchmark.md \
3286
- || ! grep -Fq 'counted by `benchmark audit` as the fourth passing pair-evidence row' benchmark/auto-resolve/README.md \
3287
- || ! grep -Fq 'counted by `benchmark audit` as the fourth passing pair-evidence row' benchmark/auto-resolve/run-real-benchmark.md \
3288
- || ! grep -Fq 'counted by `benchmark audit` as the fourth passing pair-evidence row' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
3289
- || ! grep -Fq 'the default 5-point pair margin' README.md \
3290
- || ! grep -Fq -- '`--pair-verify` and `--no-pair` are mutually exclusive' README.md \
3291
- || ! grep -Fq 'default 5-point pair margin' benchmark/auto-resolve/README.md \
3292
- || ! grep -Fq 'the default 5-point pair margin' benchmark/auto-resolve/run-real-benchmark.md \
3293
- || ! grep -Fq '3x pair/solo wall ratio' README.md \
3294
- || ! grep -Fq '3x pair/solo wall ratio' benchmark/auto-resolve/README.md \
3295
- || ! grep -Fq '3x pair/solo wall ratio' benchmark/auto-resolve/run-real-benchmark.md \
3296
- || ! grep -Fq 'audit.json` with the frontier summary' README.md \
3297
- || ! grep -Fq 'audit.json` with the frontier summary' benchmark/auto-resolve/README.md \
3298
- || ! grep -Fq 'audit.json` with the frontier summary' benchmark/auto-resolve/run-real-benchmark.md \
3299
- || ! grep -Fq 'artifact map' README.md \
3300
- || ! grep -Fq 'artifact map' benchmark/auto-resolve/README.md \
3301
- || ! grep -Fq 'artifact map' benchmark/auto-resolve/run-real-benchmark.md \
3302
- || ! grep -Fq '`checks.frontier_stdout` records summary, aggregate, final-verdict, expected, printed score-row, trigger-visible row, and hypothesis-trigger-visible row counts' README.md \
3303
- || ! grep -Fq '`headroom_rejections=...`, `pair_evidence_quality=...`,' README.md \
3304
- || ! grep -Fq '`pair_trigger_reasons=...`, `pair_evidence_hypotheses=...`, and' README.md \
3305
- || ! grep -Fq '`pair_evidence_hypothesis_triggers=...` handoff rows' README.md \
3306
- || ! grep -Fq '`pair_trigger_historical_aliases=...` when archived evidence includes legacy' README.md \
3307
- || ! grep -Fq '`pair_evidence_hypothesis_trigger_gaps=...` when documented' README.md \
3308
- || ! grep -Fq 'canonical trigger reason coverage' README.md \
3309
- || ! grep -Fq '`checks.pair_evidence_quality` records the same quality thresholds from the compact rows' README.md \
3310
- || ! grep -Fq '`checks.pair_trigger_reasons` records canonical/historical-alias/exposed/total trigger-reason row counts, fixture-level historical alias details, summary count, and row-match status' README.md \
3311
- || ! grep -Fq '`checks.pair_evidence_hypotheses` records documented/total pair-evidence hypothesis row counts' README.md \
3312
- || ! grep -Fq '`checks.pair_evidence_hypothesis_triggers` records whether documented hypotheses also appear as `spec.solo_headroom_hypothesis` trigger reasons plus fixture-level gap details' README.md \
3313
- || ! grep -Fq 'regenerated pair evidence' README.md \
3314
- || ! grep -Fq 'Historical trigger aliases are only reported for archived artifact review' README.md \
3315
- || ! grep -Fq 'current pair-evidence gates fail historical-only or unknown trigger reasons' README.md \
3316
- || ! grep -Fq '`checks.frontier_stdout`' benchmark/auto-resolve/README.md \
3317
- || ! grep -Fq 'aggregate, final-verdict, expected, printed score-row, trigger-visible row, and hypothesis-trigger-visible row counts' benchmark/auto-resolve/README.md \
3318
- || ! grep -Fq '`headroom_rejections=...`,' benchmark/auto-resolve/README.md \
3319
- || ! grep -Fq '`pair_trigger_reasons=...`, and' benchmark/auto-resolve/README.md \
3320
- || ! grep -Fq '`pair_evidence_hypotheses=...` and' benchmark/auto-resolve/README.md \
3321
- || ! grep -Fq '`pair_evidence_hypothesis_triggers=...` handoff rows' benchmark/auto-resolve/README.md \
3322
- || ! grep -Fq '`pair_trigger_historical_aliases=...` when archived evidence includes legacy' benchmark/auto-resolve/README.md \
3323
- || ! grep -Fq '`pair_evidence_hypothesis_trigger_gaps=...` when documented' benchmark/auto-resolve/README.md \
3324
- || ! grep -Fq 'canonical trigger reason coverage' benchmark/auto-resolve/README.md \
3325
- || ! grep -Fq 'records the same quality thresholds from the compact rows,' benchmark/auto-resolve/README.md \
3326
- || ! grep -Fq '`checks.pair_trigger_reasons` records canonical/historical-alias/exposed/total trigger-reason row counts, fixture-level historical alias details, summary count, and row-match status' benchmark/auto-resolve/README.md \
3327
- || ! grep -Fq '`checks.pair_evidence_hypotheses` records documented/total pair-evidence hypothesis row counts' benchmark/auto-resolve/README.md \
3328
- || ! grep -Fq '`checks.pair_evidence_hypothesis_triggers` records whether documented hypotheses also appear as `spec.solo_headroom_hypothesis` trigger reasons plus fixture-level gap details' benchmark/auto-resolve/README.md \
3329
- || ! grep -Fq 'regenerated pair evidence' benchmark/auto-resolve/README.md \
3330
- || ! grep -Fq 'Historical trigger aliases are only reported for archived artifact review' benchmark/auto-resolve/README.md \
3331
- || ! grep -Fq 'current pair-evidence gates fail historical-only or unknown trigger reasons' benchmark/auto-resolve/README.md \
3332
- || ! grep -Fq '`checks.frontier_stdout` records summary, aggregate, final-verdict, expected, printed score-row, trigger-visible row, and hypothesis-trigger-visible row counts' benchmark/auto-resolve/run-real-benchmark.md \
3333
- || ! grep -Fq 'also prints `headroom_rejections=...`, `pair_evidence_quality=...`,' benchmark/auto-resolve/run-real-benchmark.md \
3334
- || ! grep -Fq '`pair_trigger_reasons=...`, `pair_evidence_hypotheses=...`, and `pair_evidence_hypothesis_triggers=...` handoff rows' benchmark/auto-resolve/run-real-benchmark.md \
3335
- || ! grep -Fq '`pair_trigger_historical_aliases=...` when archived evidence includes legacy' benchmark/auto-resolve/run-real-benchmark.md \
3336
- || ! grep -Fq '`pair_evidence_hypothesis_trigger_gaps=...` when documented' benchmark/auto-resolve/run-real-benchmark.md \
3337
- || ! grep -Fq 'canonical trigger reason coverage' benchmark/auto-resolve/run-real-benchmark.md \
3338
- || ! grep -Fq '`checks.pair_evidence_quality` records the same quality thresholds from the compact rows' benchmark/auto-resolve/run-real-benchmark.md \
3339
- || ! grep -Fq '`checks.pair_trigger_reasons` records canonical/historical-alias/exposed/total trigger-reason row counts, fixture-level historical alias details, summary count, and row-match status' benchmark/auto-resolve/run-real-benchmark.md \
3340
- || ! grep -Fq '`checks.pair_evidence_hypotheses` records documented/total pair-evidence hypothesis row counts' benchmark/auto-resolve/run-real-benchmark.md \
3341
- || ! grep -Fq '`checks.pair_evidence_hypothesis_triggers` records whether documented hypotheses also appear as `spec.solo_headroom_hypothesis` trigger reasons plus fixture-level gap details' benchmark/auto-resolve/run-real-benchmark.md \
3342
- || ! grep -Fq 'regenerated pair evidence' benchmark/auto-resolve/run-real-benchmark.md \
3343
- || ! grep -Fq 'Historical trigger aliases are only reported for archived artifact review' benchmark/auto-resolve/run-real-benchmark.md \
3344
- || ! grep -Fq 'current pair-evidence gates fail historical-only or unknown trigger reasons' benchmark/auto-resolve/run-real-benchmark.md \
3345
- || ! grep -Fq 'compact trigger-backed verdict-bearing `pair_evidence_rows`' README.md \
3346
- || ! grep -Fq 'pair_trigger_eligible: true' README.md \
3347
- || ! grep -Fq 'non-empty `pair_trigger_reasons`, `pair_trigger_has_canonical_reason: true`, and `pair_trigger_has_hypothesis_reason`; the audit fails rows missing trigger reasons' README.md \
3348
- || ! grep -Fq 'pair_trigger_has_canonical_reason: true' README.md \
3349
- || ! grep -Fq '`pair_evidence_rows`' benchmark/auto-resolve/README.md \
3350
- || ! grep -Fq 'compact trigger-backed verdict-bearing score rows' benchmark/auto-resolve/README.md \
3351
- || ! grep -Fq 'pair_trigger_eligible: true' benchmark/auto-resolve/README.md \
3352
- || ! grep -Fq 'non-empty `pair_trigger_reasons`, `pair_trigger_has_canonical_reason: true`, and `pair_trigger_has_hypothesis_reason`; the audit fails rows missing trigger reasons' benchmark/auto-resolve/README.md \
3353
- || ! grep -Fq 'pair_trigger_has_canonical_reason: true' benchmark/auto-resolve/README.md \
3354
- || ! grep -Fq 'compact trigger-backed verdict-bearing `pair_evidence_rows`' benchmark/auto-resolve/run-real-benchmark.md \
3355
- || ! grep -Fq 'pair_trigger_eligible: true' benchmark/auto-resolve/run-real-benchmark.md \
3356
- || ! grep -Fq 'non-empty `pair_trigger_reasons`, `pair_trigger_has_canonical_reason: true`, and `pair_trigger_has_hypothesis_reason`; the audit fails rows missing trigger reasons' benchmark/auto-resolve/run-real-benchmark.md \
3357
- || ! grep -Fq 'pair_trigger_has_canonical_reason: true' benchmark/auto-resolve/run-real-benchmark.md \
3358
- || ! grep -Fq 'revalidates `pair_mode: true`' README.md \
3359
- || ! grep -Fq 'satisfy `pair_mode: true`' benchmark/auto-resolve/README.md \
3360
- || ! grep -Fq 'revalidates `pair_mode: true`' benchmark/auto-resolve/run-real-benchmark.md \
3361
- || ! grep -Fq 'JSON rows expose `pair_trigger_reasons` and' benchmark/auto-resolve/README.md \
3362
- || ! grep -Fq 'Markdown output includes a `Triggers`' benchmark/auto-resolve/README.md \
3363
- || ! grep -Fq 'trigger reasons, canonical-trigger coverage, classification counts' benchmark/auto-resolve/README.md \
3364
- || ! grep -Fq 'Its Markdown table includes a `Triggers` column' benchmark/auto-resolve/README.md \
3365
- || ! grep -Fq 'compact evidence row count must match the' README.md \
3366
- || ! grep -Fq 'compact evidence row count must match the' benchmark/auto-resolve/README.md \
3367
- || ! grep -Fq 'evidence row count must match the frontier evidence count' benchmark/auto-resolve/run-real-benchmark.md \
3368
- || ! grep -Fq '"verdict": "PASS" if unmeasured_candidate_total == 0 else "FAIL"' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
3369
- || ! grep -Fq 'assert frontier["unmeasured_count"] == 0' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3370
- || ! grep -Fq 'pair_margin_avg' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
3371
- || ! grep -Fq 'from pair_evidence_contract import' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
3372
- || ! grep -Fq 'from pair_evidence_contract import' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3373
- || ! grep -Fq 'from pair_evidence_contract import' benchmark/auto-resolve/scripts/audit-headroom-rejections.py \
3374
- || ! grep -Fq 'def normalize_pair_evidence_row' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3375
- || ! grep -Fq 'def best_pair_evidence' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3376
- || ! grep -Fq 'ALLOWED_PAIR_ARMS = {"l2_risk_probes", "l2_gated"}' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3377
- || ! grep -Fq 'pair_arm not in ALLOWED_PAIR_ARMS' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3378
- || ! grep -Fq 'pair_mode = row.get("pair_mode")' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3379
- || ! grep -Fq 'if pair_mode is not True:' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3380
- || ! grep -Fq 'pair_trigger_eligible = row.get("pair_trigger_eligible")' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3381
- || ! grep -Fq 'if pair_trigger_eligible is not True:' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3382
- || ! grep -Fq '"pair_trigger_eligible": pair_trigger_eligible' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3383
- || ! grep -Fq '"pair_mode": pair_mode' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3384
- || ! grep -Fq 'def is_strict_int' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3385
- || ! grep -Fq 'def is_score' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3386
- || ! grep -Fq '0 <= value <= 100' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3387
- || ! grep -Fq 'def is_strict_number' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3388
- || ! grep -Fq 'import math' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3389
- || ! grep -Fq 'math.isfinite(value)' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3390
- || ! grep -Fq 'and value > 0' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3391
- || ! grep -Fq 'if pair_margin != pair_score - solo_score:' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
3392
- || ! grep -Fq 'not math.isfinite(args.max_pair_solo_wall_ratio)' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
3393
- || ! grep -Fq 'nan-wall-run' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
3394
- || ! grep -Fq 'inflated-margin-run' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
3395
- || ! grep -Fq 'overrange-score-run' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
3396
- || ! grep -Fq 'invalid-arm-run' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
3397
- || ! grep -Fq 'false-pair-mode-run' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
3398
- || ! grep -Fq 'zero-wall-run' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
3399
- || ! grep -Fq 'nan-max-wall-ratio' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
3400
- || ! grep -Fq 'pair_margin_avg=+21.00 pair_margin_min=+21 wall_avg=1.28x wall_max=1.28x' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
3401
- || ! grep -Fq 'incomplete-high-run' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
3402
- || ! grep -Fq 'bad-pair-evidence-json' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
3403
- || ! grep -Fq 'pair evidence artifact malformed' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
3404
- || ! grep -Fq 'bad-pair-evidence-rows' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
3405
- || ! grep -Fq 'pair evidence artifact rows malformed' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
3406
- || ! grep -Fq 'assert len(rows["F16-cli-quote-tax-rules"]["passing_pair_evidence"]) == 1' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
3407
- || ! grep -Fq 'plus row-level verdicts' README.md \
3408
- || ! grep -Fq 'including pair arm, trigger reasons, average/minimum pair margin' README.md \
3409
- || ! grep -Fq 'Markdown frontier artifacts include a `Triggers` column' README.md \
3410
- || ! grep -Fq 'pair arm, margin, wall ratio, run id, verdict, and trigger reasons' benchmark/auto-resolve/README.md \
3411
- || ! grep -Fq 'with pair arm, verdict, and trigger reasons from the frontier step' benchmark/auto-resolve/run-real-benchmark.md \
3412
- || ! grep -Fq 'pair-candidate-frontier.py --fail-on-unmeasured' benchmark/auto-resolve/run-real-benchmark.md \
3413
- || ! grep -Fq 'audit-headroom-rejections.py' benchmark/auto-resolve/README.md \
3414
- || ! grep -Fq 'audit-headroom-rejections.py' benchmark/auto-resolve/run-real-benchmark.md \
3415
- || ! grep -Fq 'unrecorded headroom rejection(s)' benchmark/auto-resolve/scripts/audit-headroom-rejections.py \
3416
- || ! grep -Fq 'unsupported registry rejection(s)' benchmark/auto-resolve/scripts/audit-headroom-rejections.py \
3417
- || ! grep -Fq 'solo_claude={solo_score}' benchmark/auto-resolve/scripts/audit-headroom-rejections.py \
3418
- || ! grep -Fq 'expected_solo_claude={expected_solo_score}' benchmark/auto-resolve/scripts/audit-headroom-rejections.py \
3419
- || ! grep -Fq 'unsupported_registry_rejections' benchmark/auto-resolve/scripts/test-audit-headroom-rejections.sh \
3420
- || ! grep -Fq 'expected_solo_claude=98' benchmark/auto-resolve/scripts/test-audit-headroom-rejections.sh \
3421
- || ! grep -Fq 'active registry entries whose reason cites a run id or' benchmark/auto-resolve/README.md \
3422
- || ! grep -Fq 'active rejected-registry reason is backed' benchmark/auto-resolve/run-real-benchmark.md \
3423
- || ! grep -Fq 'invalid headroom rejections' bin/devlyn.js \
3424
- || ! grep -Fq 'MALFORMED_JSON' benchmark/auto-resolve/scripts/audit-headroom-rejections.py \
3425
- || ! grep -Fq 'MALFORMED_ROWS' benchmark/auto-resolve/scripts/audit-headroom-rejections.py \
3426
- || ! grep -Fq 'F33-cli-new-candidate' benchmark/auto-resolve/scripts/test-audit-headroom-rejections.sh \
3427
- || ! grep -Fq 'bad-json-headroom <unknown>' benchmark/auto-resolve/scripts/test-audit-headroom-rejections.sh \
3428
- || ! grep -Fq 'malformed-headroom <unknown>' benchmark/auto-resolve/scripts/test-audit-headroom-rejections.sh \
3429
- || ! grep -Fq 'candidate_unmeasured' benchmark/auto-resolve/run-real-benchmark.md \
3430
- || ! grep -Fq 'pair_evidence_passed' benchmark/auto-resolve/run-real-benchmark.md \
3431
- || ! grep -Fq 'print_summary(report)' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
3432
- || ! grep -Fq 'fail_on_unmeasured' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
3433
- || ! grep -Fq 'unmeasured candidate fixture(s)' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
3434
- || ! grep -Fq 'pure JSON stdout must not include final text verdict' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
3435
- || ! grep -Fq 'benchmark frontier pure JSON stdout must not include final text verdict' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3436
- || ! grep -Fq -- '--fail-on-unmeasured' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3437
- || ! grep -Fq -- '--fail-on-unmeasured' bin/devlyn.js \
3438
- || ! grep -Fq -- '--fail-on-unmeasured' benchmark/auto-resolve/README.md \
3439
- || ! grep -Fq -- '--fail-on-unmeasured' benchmark/auto-resolve/run-real-benchmark.md \
3440
- || ! grep -Fq 'bare={bare} solo_claude={solo} pair={pair} arm={arm} margin={margin}' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
3441
- || ! grep -Fq 'verdict=pair_evidence_passed' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
3442
- || ! grep -Fq 'F16-cli-quote-tax-rules: bare=50 solo_claude=75 pair=96 arm=l2_risk_probes margin=+21' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
3443
- || ! grep -Fq 'verdict=pair_evidence_passed' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
3444
- || ! grep -Fq '[audit] frontier' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3445
- || ! grep -Fq 'fixtures=21 rejected=17 candidates=4 pair_evidence=4 unmeasured=0 verdict=PASS' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3446
- || ! grep -Fq 'F16-cli-quote-tax-rules: bare=50 solo_claude=75 pair=96 arm=l2_risk_probes margin=+21' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3447
- || ! grep -Fq 'frontier.stdout' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3448
- || ! grep -Fq 'headroom-rejections.stdout' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3449
- || ! grep -Fq '"frontier_stdout": "frontier.stdout"' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3450
- || ! grep -Fq '"headroom_rejections_stdout": "headroom-rejections.stdout"' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3451
- || ! grep -Fq 'def load_headroom_audit_summary' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3452
- || ! grep -Fq 'def check_headroom_audit_report' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3453
- || ! grep -Fq 'def print_headroom_rejections_summary' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3454
- || ! grep -Fq 'headroom_rejections={status} verdict={verdict}' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3455
- || ! grep -Fq 'headroom_report_status == 0' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3456
- || ! grep -Fq '"report_check_exit_code": headroom_report_status' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3457
- || ! grep -Fq 'headroom audit unsupported registry rejection count missing or malformed' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3458
- || ! grep -Fq 'headroom_rejections=PASS verdict=PASS unrecorded=0 unsupported=0' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3459
- || ! grep -Fq 'headroom_rejections=PASS verdict=PASS unrecorded=0 unsupported=0' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3460
- || ! grep -Fq 'pair trigger reason rows {len(rows)} do not match summary count {count}' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3461
- || ! grep -Fq 'pair trigger reasons missing for fixture(s)' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3462
- || ! grep -Fq 'missing-trigger-reasons' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3463
- || ! grep -Fq 'malformed-trigger-reason-rows' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3464
- || ! grep -Fq 'mixed-unknown-trigger-reason-rows' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3465
- || ! grep -Fq 'pair_trigger_reasons=PASS canonical=2 historical_alias=1 exposed=2 total=2 summary=2 rows_match=true' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3466
- || ! grep -Fq 'pair_trigger_historical_aliases=F21-cli-scheduler-priority=risk_profile.high_risk' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3467
- || ! grep -Fq '"unsupported_registry_rejection_count": (' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3468
- || ! grep -Fq 'len(unsupported) if isinstance(unsupported, list) else None' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3469
- || ! grep -Fq 'headroom-missing-unsupported' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3470
- || ! grep -Fq 'unsupported_registry_rejection_count' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3471
- || ! grep -Fq 'unsupported_registry_rejection_count' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3472
- || ! grep -Fq '`checks.headroom_rejections` records child verdict plus unrecorded/unsupported counts' README.md \
3473
- || ! grep -Fq '`checks.headroom_rejections` records the child verdict plus unrecorded and' benchmark/auto-resolve/README.md \
3474
- || ! grep -Fq '`checks.headroom_rejections` records child verdict plus unrecorded/unsupported counts' benchmark/auto-resolve/run-real-benchmark.md \
3475
- || ! grep -Fq 'def check_frontier_stdout' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3476
- || ! grep -Fq '"final_verdict_rows": final_verdict_rows' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3477
- || ! grep -Fq 'frontier stdout final verdict row count is not exactly 1' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3478
- || ! grep -Fq 'frontier stdout missing score row' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3479
- || ! grep -Fq 'wall={wall} run={run_id}' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3480
- || ! grep -Fq 'frontier stdout aggregate score row count is not exactly 1' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3481
- || ! grep -Fq 'frontier stdout summary score row count is not exactly 1' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3482
- || ! grep -Fq 'frontier stdout aggregate fields malformed' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3483
- || ! grep -Fq 'frontier stdout score row count' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3484
- || ! grep -Fq 'summary_count = stdout.splitlines().count(required_summary)' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3485
- || ! grep -Fq 'aggregate_count = stdout.splitlines().count(required_aggregate)' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3486
- || ! grep -Fq 'frontier stdout summary counts malformed' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3487
- || ! grep -Fq 'count_keys = {' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3488
- || ! grep -Fq 'pair_margin_avg={avg} pair_margin_min={min_margin}' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3489
- || ! grep -Fq 'def format_decimal_margin' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3490
- || ! grep -Fq 'def format_wall_ratio' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3491
- || ! grep -Fq 'frontier stdout check missing summary fields' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3492
- || ! grep -Fq 'missing-frontier-score-row' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3493
- || ! grep -Fq 'missing-frontier-aggregate-row' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3494
- || ! grep -Fq 'duplicate-frontier-summary-row' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3495
- || ! grep -Fq 'duplicate-frontier-aggregate-row' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3496
- || ! grep -Fq 'pair_margin_avg=+27.00 pair_margin_min=+21 wall_avg=1.38x wall_max=1.47x' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3497
- || ! grep -Fq 'partial-frontier-score-row' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3498
- || ! grep -Fq 'extra-frontier-score-row' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3499
- || ! grep -Fq 'malformed-frontier-stdout-summary' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3500
- || ! grep -Fq 'malformed-frontier-stdout-counts' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3501
- || ! grep -Fq 'malformed-frontier-stdout-aggregate' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3502
- || ! grep -Fq 'audit["checks"]["frontier_stdout"]["status"] == "PASS"' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3503
- || ! grep -Fq 'audit["checks"]["frontier_stdout"]["summary_rows"] == 1' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3504
- || ! grep -Fq 'report["checks"]["frontier_stdout"]["aggregate_rows"] == 1' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3505
- || ! grep -Fq 'audit["checks"]["frontier_stdout"]["final_verdict_rows"] == 1' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3506
- || ! grep -Fq 'missing-final-frontier-verdict' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3507
- || ! grep -Fq 'duplicate-final-frontier-verdict' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3508
- || ! grep -Fq 'audit["checks"]["frontier_stdout"]["expected_rows"] == len(audit["pair_evidence_rows"])' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3509
- || ! grep -Fq 'audit["checks"]["frontier_stdout"]["trigger_rows"] == len(audit["pair_evidence_rows"])' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3510
- || ! grep -Fq 'audit["checks"]["frontier_stdout"]["hypothesis_trigger_rows"] == len(audit["pair_evidence_rows"])' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3511
- || ! grep -Fq 'audit["checks"]["frontier_stdout"]["trigger_rows_match_count"] is True' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3512
- || ! grep -Fq 'audit["checks"]["frontier_stdout"]["hypothesis_trigger_rows_match_count"] is True' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3513
- || ! grep -Fq 'report["checks"]["frontier_stdout"]["trigger_rows"] == 2' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3514
- || ! grep -Fq 'report["checks"]["frontier_stdout"]["hypothesis_trigger_rows"] == 2' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3515
- || ! grep -Fq 'report["checks"]["frontier_stdout"]["trigger_rows_match_count"] is True' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3516
- || ! grep -Fq 'report["checks"]["frontier_stdout"]["hypothesis_trigger_rows_match_count"] is True' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3517
- || ! grep -Fq 'report["checks"]["frontier_stdout"]["rows_match_count"] is True' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3518
- || ! grep -Fq 'audit["artifacts"]["frontier_stdout"] == "frontier.stdout"' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3519
- || ! grep -Fq 'report["artifacts"] == {' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3520
- || ! grep -Fq 'frontier.stdout' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
3521
- || ! grep -Fq 'frontier.stdout' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
3522
- || ! grep -Fq 'frontier summary and an artifact map (`artifacts`)' benchmark/auto-resolve/README.md \
3523
- || ! grep -Fq 'artifact map' benchmark/auto-resolve/run-real-benchmark.md \
3524
- || ! grep -Fq 'child stdout/stderr logs' benchmark/auto-resolve/run-real-benchmark.md \
3525
- || ! grep -Fq '| Fixture | Status | Verdict | Evidence | Pair arm | Triggers | Hypothesis trigger | Bare | Solo_claude | Pair | Margin | Wall ratio | Rejected reason |' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
3526
- || ! grep -Fq 'benchmark frontier` also prints a stdout score summary for existing complete pair' README.md \
3527
- || ! grep -Fq 'plus row-level verdicts' README.md \
3528
- || ! grep -Fq 'including pair arm, trigger reasons, average/minimum pair margin' README.md \
3529
- || ! grep -Fq 'Markdown frontier artifacts include a `Triggers` column' README.md \
3530
- || ! grep -Fq 'Full-pipeline pair gate artifacts record `require_hypothesis_trigger` in JSON' README.md \
3531
- || ! grep -Fq 'Full-pipeline pair gate artifacts record `require_hypothesis_trigger` in JSON' benchmark/auto-resolve/README.md \
3532
- || ! grep -Fq 'Full-pipeline pair gate artifacts record `require_hypothesis_trigger` in JSON' benchmark/auto-resolve/run-real-benchmark.md \
3533
- || ! grep -Fq 'includes a Markdown `Hypothesis trigger` column' benchmark/auto-resolve/README.md \
3534
- || ! grep -Fq 'includes a Markdown `Hypothesis trigger` column' benchmark/auto-resolve/run-real-benchmark.md \
3535
- || ! grep -Fq 'pair arm, margin, wall ratio, run id, verdict, and trigger reasons' benchmark/auto-resolve/README.md \
3536
- || ! grep -Fq 'overall verdict plus row-level verdict, pair-arm, and trigger-reason columns' benchmark/auto-resolve/README.md \
3537
- || ! grep -Fq 'with pair arm, verdict, and trigger reasons from the frontier step' benchmark/auto-resolve/run-real-benchmark.md \
3538
- || ! grep -Fq 'overall verdict plus row-level verdict, pair-arm, and trigger-reason columns' benchmark/auto-resolve/run-real-benchmark.md \
3539
- || ! grep -Fq 'complete pair evidence rows' benchmark/auto-resolve/README.md \
3540
- || ! grep -Fq 'complete pair evidence rows' benchmark/auto-resolve/run-real-benchmark.md; then
3541
- offenders="${offenders}"$'\n'"benchmark docs and CLI must expose the pair-candidate frontier report before new provider spend"
3542
- fi
3543
- if ! grep -Fq '"benchmark/auto-resolve/scripts/**"' package.json \
3544
- || ! grep -Fq 'benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.json' package.json \
3545
- || ! grep -Fq 'benchmark/auto-resolve/results/20260512-f2-medium-headroom/headroom-gate.json' package.json \
3546
- || ! grep -Fq 'benchmark/auto-resolve/results/20260512-f31-seat-rebalance-headroom/headroom-gate.json' package.json \
3547
- || ! grep -Fq 'benchmark/auto-resolve/results/20260512-f32-subscription-renewal-headroom/headroom-gate.json' package.json; then
3548
- offenders="${offenders}"$'\n'"package.json must include benchmark scripts, runner regression tests, current pair evidence, and rejected-headroom evidence in npm packages"
3549
- fi
3550
- if ! grep -Fq '"max_observed_pair_solo_wall_ratio": 2.2506234413965087' benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.json \
3551
- || ! grep -Fq '"max_observed_pair_solo_wall_ratio": 1.4728476821192054' benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.json \
3552
- || ! grep -Fq 'pair_trigger eligible with a canonical reason' benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.json \
3553
- || ! grep -Fq 'pair_trigger eligible with a canonical reason' benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.json \
3554
- || ! grep -Fq 'pair_trigger eligible with canonical reason' benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.md \
3555
- || ! grep -Fq 'pair_trigger eligible with canonical reason' benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.md \
3556
- || ! grep -Fq 'Allowed pair/solo wall ratio: 3.00x' benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.md \
3557
- || ! grep -Fq 'Maximum observed pair/solo wall ratio: 2.25x' benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.md \
3558
- || ! grep -Fq 'Allowed pair/solo wall ratio: 3.00x' benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.md \
3559
- || ! grep -Fq 'Maximum observed pair/solo wall ratio: 1.47x' benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.md; then
3560
- offenders="${offenders}"$'\n'"packaged pair evidence artifacts must use the current observed-vs-allowed wall-ratio schema and canonical trigger rule wording"
3561
- fi
3562
- if make_temp_dir package_results /tmp/devlyn-lint-package-results.XXXXXX \
3563
- && make_temp_dir package_audit /tmp/devlyn-lint-package-audit.XXXXXX \
3564
- && make_temp_file package_audit_stdout /tmp/devlyn-lint-package-audit.out.XXXXXX; then
3565
- cp -R benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof "$package_results/"
3566
- cp -R benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1 "$package_results/"
3567
- for rejected_run in \
3568
- 20260507-f10-f11-tier1-full-pipeline \
3569
- 20260508-f22-exact-error-headroom \
3570
- 20260508-f26-headroom \
3571
- 20260511-f3-http-error-headroom \
3572
- 20260511-f12-webhook-headroom \
3573
- 20260511-f15-concurrency-headroom \
3574
- 20260512-f2-medium-headroom \
3575
- 20260512-f4-web-headroom \
3576
- 20260512-f5-fixloop-headroom \
3577
- 20260512-f6-checksum-headroom \
3578
- 20260512-f7-scope-headroom \
3579
- 20260512-f9-e2e-headroom \
3580
- 20260512-f31-seat-rebalance-headroom \
3581
- 20260512-f32-subscription-renewal-headroom; do
3582
- cp -R "benchmark/auto-resolve/results/$rejected_run" "$package_results/"
3583
- done
3584
- if ! python3 benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3585
- --results-root "$package_results" \
3586
- --out-dir "$package_audit" >"$package_audit_stdout" 2>&1; then
3587
- offenders="${offenders}"$'\n'"packaged pair evidence subset must pass benchmark audit without relying on unshipped local results"
3588
- elif ! grep -Fq 'headroom_rejections=PASS verdict=PASS unrecorded=0 unsupported=0' "$package_audit_stdout" \
3589
- || ! grep -Fq 'pair_evidence_quality=PASS min_pair_margin_actual=+21 min_pair_margin_required=+5 max_wall_actual=2.25x max_wall_allowed=3.00x' "$package_audit_stdout" \
3590
- || ! grep -Fq 'pair_trigger_reasons=PASS canonical=4 historical_alias=0 exposed=4 total=4 summary=4 rows_match=true' "$package_audit_stdout" \
3591
- || ! grep -Fq 'pair_evidence_hypothesis_triggers=PASS matched=4 documented=4 total=4' "$package_audit_stdout" \
3592
- || grep -Fq 'pair_trigger_historical_aliases=' "$package_audit_stdout" \
3593
- || grep -Fq 'pair_evidence_hypothesis_trigger_gaps=' "$package_audit_stdout"; then
3594
- offenders="${offenders}"$'\n'"packaged pair evidence subset audit stdout must expose headroom, pair-quality, and trigger-reason handoff rows"
3595
- elif ! python3 - "$package_audit/audit.json" <<'PY'
3596
- import json
3597
- import sys
3598
-
3599
- report = json.load(open(sys.argv[1], encoding="utf8"))
3600
- rows = report.get("pair_evidence_rows")
3601
- frontier_summary = report.get("frontier_summary", {})
3602
- frontier_report = report.get("checks", {}).get("frontier_report", {})
3603
- frontier_stdout = report.get("checks", {}).get("frontier_stdout", {})
3604
- min_pair_evidence = report.get("checks", {}).get("min_pair_evidence", {})
3605
- pair_evidence_quality = report.get("checks", {}).get("pair_evidence_quality", {})
3606
- pair_trigger_reasons = report.get("checks", {}).get("pair_trigger_reasons", {})
3607
- headroom_rejections = report.get("checks", {}).get("headroom_rejections", {})
3608
- artifacts = report.get("artifacts", {})
3609
- assert report.get("verdict") == "PASS"
3610
- assert frontier_summary.get("verdict") == "PASS"
3611
- assert frontier_summary.get("pair_evidence_count") == 4
3612
- assert frontier_summary.get("unmeasured_count") == 0
3613
- assert frontier_report.get("status") == "PASS"
3614
- assert frontier_report.get("verdict") == "PASS"
3615
- assert frontier_report.get("unmeasured_count") == 0
3616
- assert frontier_stdout.get("status") == "PASS"
3617
- assert frontier_stdout.get("report", "").endswith("frontier.stdout")
3618
- assert frontier_stdout.get("summary_rows") == 1
3619
- assert frontier_stdout.get("aggregate_rows") == 1
3620
- assert frontier_stdout.get("final_verdict_rows") == 1
3621
- assert frontier_stdout.get("expected_rows") == len(rows) == 4
3622
- assert frontier_stdout.get("stdout_rows") == len(rows) == 4
3623
- assert frontier_stdout.get("trigger_rows") == len(rows) == 4
3624
- assert frontier_stdout.get("hypothesis_trigger_rows") == len(rows) == 4
3625
- assert frontier_stdout.get("rows_match_count") is True
3626
- assert frontier_stdout.get("trigger_rows_match_count") is True
3627
- assert frontier_stdout.get("hypothesis_trigger_rows_match_count") is True
3628
- assert headroom_rejections.get("status") == "PASS"
3629
- assert headroom_rejections.get("report_check_exit_code") == 0
3630
- assert headroom_rejections.get("verdict") == "PASS"
3631
- assert headroom_rejections.get("unrecorded_failure_count") == 0
3632
- assert headroom_rejections.get("unsupported_registry_rejection_count") == 0
3633
- assert min_pair_evidence.get("rows_match_count") is True
3634
- assert min_pair_evidence.get("actual_rows") == len(rows) == 4
3635
- assert pair_evidence_quality.get("status") == "PASS"
3636
- assert pair_evidence_quality.get("min_pair_margin_actual") == frontier_summary.get("pair_margin_min")
3637
- assert pair_evidence_quality.get("max_pair_solo_wall_ratio_actual") == frontier_summary.get("pair_solo_wall_ratio_max")
3638
- assert pair_trigger_reasons.get("status") == "PASS"
3639
- assert pair_trigger_reasons.get("summary_pair_evidence_count") == 4
3640
- assert pair_trigger_reasons.get("canonical_rows") == 4
3641
- assert pair_trigger_reasons.get("historical_alias_rows") == 0
3642
- assert pair_trigger_reasons.get("historical_alias_details") == []
3643
- assert pair_trigger_reasons.get("exposed_rows") == 4
3644
- assert pair_trigger_reasons.get("total_rows") == 4
3645
- assert pair_trigger_reasons.get("rows_match_count") is True
3646
- pair_hypothesis_triggers = report.get("checks", {}).get("pair_evidence_hypothesis_triggers", {})
3647
- assert pair_hypothesis_triggers.get("status") == "PASS"
3648
- assert pair_hypothesis_triggers.get("matched_rows") == 4
3649
- assert pair_hypothesis_triggers.get("documented_rows") == 4
3650
- assert pair_hypothesis_triggers.get("gap_details") == []
3651
- assert artifacts.get("frontier_json") == "frontier.json"
3652
- assert artifacts.get("frontier_stdout") == "frontier.stdout"
3653
- assert artifacts.get("headroom_audit_json") == "headroom-audit.json"
3654
- assert artifacts.get("headroom_rejections_stdout") == "headroom-rejections.stdout"
3655
- for row in rows:
3656
- assert row.get("verdict") == "pair_evidence_passed"
3657
- assert row.get("pair_arm") == "l2_risk_probes"
3658
- assert row.get("pair_mode") is True
3659
- assert row.get("pair_trigger_eligible") is True
3660
- assert isinstance(row.get("pair_trigger_reasons"), list)
3661
- assert row.get("pair_trigger_reasons")
3662
- assert row.get("pair_trigger_has_canonical_reason") is True
3663
- assert row.get("pair_trigger_has_hypothesis_reason") is True
3664
- PY
3665
- then
3666
- offenders="${offenders}"$'\n'"packaged pair evidence subset audit.json must expose frontier_report, frontier_stdout, artifact map, and 4 trigger-backed verdict-bearing pair rows with trigger reasons"
3667
- fi
3668
- rm -rf "$package_results" "$package_audit" "$package_audit_stdout"
3669
- else
3670
- offenders="${offenders}"$'\n'"packaged pair evidence subset could not allocate temporary audit workspace"
3671
- fi
3672
- if make_temp_file pack_json /tmp/devlyn-lint-pack.json.XXXXXX \
3673
- && make_temp_dir pack_cache /tmp/devlyn-lint-pack-cache.XXXXXX; then
3674
- if npm_config_cache="$pack_cache" npm pack --dry-run --json > "$pack_json" 2>/dev/null; then
3675
- if ! node - "$pack_json" <<'NODE'
3676
- const fs = require("fs");
3677
- const path = require("path");
3678
- const packPath = process.argv[2];
3679
- const pack = JSON.parse(fs.readFileSync(packPath, "utf8"))[0];
3680
- const files = new Set(pack.files.map((file) => file.path));
3681
- function listFiles(dir) {
3682
- return fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
3683
- const full = path.join(dir, entry.name);
3684
- if (entry.isDirectory()) return listFiles(full);
3685
- if (!entry.isFile()) return [];
3686
- return [full.split(path.sep).join("/")];
3687
- });
3688
- }
3689
- const shadowRequired = fs
3690
- .readdirSync("benchmark/auto-resolve/shadow-fixtures", { withFileTypes: true })
3691
- .filter((entry) => entry.isDirectory() && /^S/.test(entry.name))
3692
- .flatMap((entry) => listFiles(path.join("benchmark/auto-resolve/shadow-fixtures", entry.name)));
3693
- const required = [
3694
- "benchmark/auto-resolve/BENCHMARK-RESULTS.md",
3695
- "benchmark/auto-resolve/run-real-benchmark.md",
3696
- "benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.md",
3697
- "benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.json",
3698
- "benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.md",
3699
- "benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.json",
3700
- "benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F16-cli-quote-tax-rules/l2_risk_probes/result.json",
3701
- "benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F23-cli-fulfillment-wave/l2_risk_probes/result.json",
3702
- "benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F25-cli-cart-promotion-rules/l2_risk_probes/result.json",
3703
- "benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.md",
3704
- "benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.json",
3705
- "benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.md",
3706
- "benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.json",
3707
- "benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/F21-cli-scheduler-priority/l2_risk_probes/result.json",
3708
- "benchmark/auto-resolve/results/20260512-f2-medium-headroom/headroom-gate.json",
3709
- "benchmark/auto-resolve/results/20260512-f31-seat-rebalance-headroom/headroom-gate.json",
3710
- "benchmark/auto-resolve/results/20260512-f32-subscription-renewal-headroom/headroom-gate.json",
3711
- "benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/NOTES.md",
3712
- "benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/spec.md",
3713
- "benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/expected.json",
3714
- "benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/metadata.json",
3715
- "benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/verifiers/priority-transfer-rollback.js",
3716
- "benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/verifiers/duplicate-event-error.js",
3717
- "benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/NOTES.md",
3718
- "benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/spec.md",
3719
- "benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/expected.json",
3720
- "benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/metadata.json",
3721
- "benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/verifiers/priority-credit-rollback.js",
3722
- "benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/verifiers/duplicate-renewal-error.js",
3723
- "benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh",
3724
- "benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh",
3725
- "benchmark/auto-resolve/scripts/pair_evidence_contract.py",
3726
- "benchmark/auto-resolve/scripts/pair-candidate-frontier.py",
3727
- "benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh",
3728
- "benchmark/auto-resolve/scripts/audit-pair-evidence.py",
3729
- "benchmark/auto-resolve/scripts/solo-headroom-hypothesis.py",
3730
- "benchmark/auto-resolve/scripts/solo-ceiling-avoidance.py",
3731
- "benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh",
3732
- "benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh",
3733
- "benchmark/auto-resolve/scripts/test-check-f9-artifacts.sh",
3734
- "benchmark/auto-resolve/scripts/test-lint-fixtures.sh",
3735
- "benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh",
3736
- "benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh",
3737
- "benchmark/auto-resolve/scripts/test-run-swebench-solver-batch.sh",
3738
- "benchmark/auto-resolve/scripts/test-ship-gate.sh",
3739
- "benchmark/auto-resolve/scripts/test-iter-0033c-l1-summary.sh",
3740
- "benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh",
3741
- "benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/RETIRED.md",
3742
- "benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/spec.md",
3743
- "benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/RETIRED.md",
3744
- "benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/spec.md",
3745
- "benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/RETIRED.md",
3746
- "benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/spec.md",
3747
- "benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/RETIRED.md",
3748
- "scripts/lint-fixtures.sh",
3749
- "scripts/lint-shadow-fixtures.sh",
3750
- ...shadowRequired,
3751
- ];
3752
- const missing = required.filter((file) => !files.has(file));
3753
- if (missing.length > 0) {
3754
- console.error(missing.join("\n"));
3755
- process.exit(1);
3756
- }
3757
- const forbidden = pack.files
3758
- .map((file) => file.path)
3759
- .filter((file) => file.includes("__pycache__") || file.endsWith(".pyc"));
3760
- if (forbidden.length > 0) {
3761
- console.error(forbidden.join("\n"));
3762
- process.exit(2);
3763
- }
3764
- NODE
3765
- then
3766
- offenders="${offenders}"$'\n'"npm pack dry-run must include benchmark runner/gate regression tests, all shadow fixture files, retired fixture replay docs, and exclude pycache artifacts"
3767
- fi
3768
- else
3769
- offenders="${offenders}"$'\n'"npm pack dry-run failed while checking benchmark runner/gate regression tests"
3770
- fi
3771
- else
3772
- offenders="${offenders}"$'\n'"npm pack dry-run could not allocate temporary package workspace"
3773
- fi
3774
- [ -n "${pack_json:-}" ] && rm -f "$pack_json"
3775
- [ -n "${pack_cache:-}" ] && rm -rf "$pack_cache"
3776
- non_executable_shell_scripts=$(find benchmark/auto-resolve/scripts -maxdepth 1 -name '*.sh' -type f ! -perm -111 -print | sort)
3777
- if [ -n "$non_executable_shell_scripts" ]; then
3778
- while IFS= read -r f; do
3779
- offenders="${offenders}"$'\n'"benchmark shell script must be executable: $f"
3780
- done <<< "$non_executable_shell_scripts"
3781
- fi
3782
- for gate_test in \
3783
- test-benchmark-arg-parsing.sh \
3784
- test-pair-candidate-frontier.sh \
3785
- test-audit-pair-evidence.sh \
3786
- test-audit-headroom-rejections.sh \
3787
- test-build-pair-eligible-manifest.sh \
3788
- test-ship-gate.sh \
3789
- test-headroom-gate.sh \
3790
- test-run-headroom-candidate.sh \
3791
- test-check-f9-artifacts.sh \
3792
- test-lint-fixtures.sh \
3793
- test-run-full-pipeline-pair-candidate.sh \
3794
- test-full-pipeline-pair-gate.sh \
3795
- test-iter-0033c-l1-summary.sh \
3796
- test-iter-0033c-compare.sh \
3797
- test-run-swebench-solver-batch.sh \
3798
- test-swebench-frozen-case.sh \
3799
- test-frozen-verify-gate.sh
3800
- do
3801
- if ! grep -Fq "$gate_test" benchmark/auto-resolve/README.md; then
3802
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/README.md: gate-change instructions must list $gate_test"
3803
- fi
3804
- done
3805
- if ! bash benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh >/dev/null 2>&1; then
3806
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh: failed"
3807
- fi
3808
- if ! bash benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh >/dev/null 2>&1; then
3809
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh: failed"
3810
- fi
3811
- if ! bash benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh >/dev/null 2>&1; then
3812
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh: failed"
3813
- fi
3814
- if ! bash benchmark/auto-resolve/scripts/test-audit-headroom-rejections.sh >/dev/null 2>&1; then
3815
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-audit-headroom-rejections.sh: failed"
3816
- fi
3817
- if ! bash benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh >/dev/null 2>&1; then
3818
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh: failed"
3819
- fi
3820
- if ! bash benchmark/auto-resolve/scripts/test-headroom-gate.sh >/dev/null 2>&1; then
3821
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-headroom-gate.sh: failed"
3822
- fi
3823
- if ! bash benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh >/dev/null 2>&1; then
3824
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh: failed"
3825
- fi
3826
- if ! bash benchmark/auto-resolve/scripts/test-check-f9-artifacts.sh >/dev/null 2>&1; then
3827
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-check-f9-artifacts.sh: failed"
3828
- fi
3829
- if ! bash benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh >/dev/null 2>&1; then
3830
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh: failed"
3831
- fi
3832
- if ! bash benchmark/auto-resolve/scripts/test-ship-gate.sh >/dev/null 2>&1; then
3833
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-ship-gate.sh: failed"
3834
- fi
3835
- if ! bash benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh >/dev/null 2>&1; then
3836
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh: failed"
3837
- fi
3838
- if ! bash benchmark/auto-resolve/scripts/test-iter-0033c-l1-summary.sh >/dev/null 2>&1; then
3839
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-iter-0033c-l1-summary.sh: failed"
3840
- fi
3841
- if ! bash benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh >/dev/null 2>&1; then
3842
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh: failed"
3843
- fi
3844
- if ! bash benchmark/auto-resolve/scripts/test-run-swebench-solver-batch.sh >/dev/null 2>&1; then
3845
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-run-swebench-solver-batch.sh: failed"
3846
- fi
3847
- if ! bash benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh >/dev/null 2>&1; then
3848
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh: failed"
3849
- fi
3850
- if ! bash benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh >/dev/null 2>&1; then
3851
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh: failed"
3852
- fi
3853
- if ! bash scripts/lint-fixtures.sh >/dev/null 2>&1; then
3854
- offenders="${offenders}"$'\n'"scripts/lint-fixtures.sh: failed"
3855
- fi
3856
- if ! bash scripts/lint-shadow-fixtures.sh >/dev/null 2>&1; then
3857
- offenders="${offenders}"$'\n'"scripts/lint-shadow-fixtures.sh: failed"
3858
- fi
3859
- if ! bash benchmark/auto-resolve/scripts/test-lint-fixtures.sh >/dev/null 2>&1; then
3860
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-lint-fixtures.sh: failed"
3861
- fi
3862
- if ! python3 -m py_compile benchmark/auto-resolve/scripts/solo-headroom-hypothesis.py >/dev/null 2>&1; then
3863
- offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/solo-headroom-hypothesis.py: py_compile failed"
3864
- fi
3865
- if ! grep -Fq 'high-risk fixture must include a resolve risk-trigger term' scripts/lint-fixtures.sh \
3866
- || ! grep -Fq 'SOLO_HEADROOM_CHECK="$REPO_ROOT/benchmark/auto-resolve/scripts/solo-headroom-hypothesis.py"' scripts/lint-fixtures.sh \
3867
- || ! grep -Fq 'pair_evidence_passed fixture spec.md must document an actionable solo-headroom hypothesis with solo_claude miss and observable command from expected.json' scripts/lint-fixtures.sh \
3868
- || ! grep -Fq -- '--expected-json "$d/expected.json" "$d/spec.md"' scripts/lint-fixtures.sh \
3869
- || ! grep -Fq 'str(fixture_dir / "expected.json")' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
3870
- || ! grep -Fq 'hypothesis command must match expected.json verification command' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
3871
- || ! grep -Fq 'pair-evidence-hypothesis-fail.out' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
3872
- || ! grep -Fq 'pair-evidence-hypothesis-pass.out' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
3873
- || ! grep -Fq 'DEVLYN_FIXTURES_DIR' scripts/lint-fixtures.sh \
3874
- || ! grep -Fq 'DEVLYN_FIXTURE_GLOB' scripts/lint-fixtures.sh \
3875
- || ! grep -Fq 'DEVLYN_FIXTURE_GLOB="S*"' scripts/lint-shadow-fixtures.sh \
3876
- || ! grep -Fq 'benchmark/auto-resolve/shadow-fixtures' scripts/lint-shadow-fixtures.sh \
3877
- || ! grep -Fq 'category") != "high-risk"' scripts/lint-fixtures.sh \
3878
- || ! grep -Fq 'permissions?' scripts/lint-fixtures.sh \
3879
- || ! grep -Fq 'idempoten\w*' scripts/lint-fixtures.sh \
3880
- || ! grep -Fq 'output-shape' scripts/lint-fixtures.sh \
3881
- || ! grep -Fq 'high-risk trigger validation' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
3882
- || ! grep -Fq 'DEVLYN_FIXTURES_DIR="$FIXTURES_DIR" bash "$ROOT/scripts/lint-fixtures.sh"' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
3883
- || ! grep -Fq 'DEVLYN_LINT_FIXTURES_NO_JSONSCHEMA=1' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
3884
- || ! grep -Fq 'spec-verify-check --check-expected failed' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
3885
- || ! grep -Fq 'unless spec.expected.json declares "pure_design": true' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
3886
- || ! grep -Fq 'verification_commands must be an array' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
3887
- || ! grep -Fq 'expected.json must be an object' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
3888
- || ! grep -Fq 'hidden oracle missing contract_refs' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
3889
- || ! grep -Fq 'contract_ref not found in spec.md' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
3890
- || ! grep -Fq 'BENCH_FIXTURE_DIR file not found' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
3891
- || ! grep -Fq 'BENCH_FIXTURE_DIR file escapes fixture dir' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
3892
- || ! grep -Fq 'hidden oracle must reference an explicit $BENCH_FIXTURE_DIR/... file' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
3893
- || ! grep -Fq 'hidden oracle must assert stdout_contains includes' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
3894
- || ! grep -Fq 'cd \"$BENCH_FIXTURE_DIR\" && node verifiers/hidden-oracle.js' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
3895
- || ! grep -Fq 'missing-hidden-oracle.js' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
3896
- || ! grep -Fq '../outside-hidden-oracle.js' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
3897
- || ! grep -Fq 'This visible contract is not in the spec.' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
3898
- || ! grep -Fq 'spec-verify-check --check failed' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
3899
- || ! grep -Fq 'frontmatter complexity must be one of' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
3900
- || ! grep -Fq 'resolve spec contract enum' benchmark/auto-resolve/fixtures/SCHEMA.md \
3901
- || ! grep -Fq 'metadata.difficulty' benchmark/auto-resolve/fixtures/SCHEMA.md \
3902
- || ! grep -Fq "grep -Fq 'Traceback'" benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
3903
- || ! grep -Fq 'hidden oracle missing contract_refs' scripts/lint-fixtures.sh \
3904
- || ! grep -Fq 'contract_ref not found in spec.md' scripts/lint-fixtures.sh \
3905
- || ! grep -Fq 'BENCH_FIXTURE_DIR file not found' scripts/lint-fixtures.sh \
3906
- || ! grep -Fq 'BENCH_FIXTURE_DIR file escapes fixture dir' scripts/lint-fixtures.sh \
3907
- || ! grep -Fq 'hidden oracle must reference an explicit $BENCH_FIXTURE_DIR/... file' scripts/lint-fixtures.sh \
3908
- || ! grep -Fq 'hidden oracle must assert stdout_contains includes' scripts/lint-fixtures.sh \
3909
- || ! grep -Fq 'SPEC_VERIFY_CHECK="$REPO_ROOT/config/skills/_shared/spec-verify-check.py"' scripts/lint-fixtures.sh \
3910
- || ! grep -Fq 'python3 "$SPEC_VERIFY_CHECK" --check "$d/spec.md"' scripts/lint-fixtures.sh \
3911
- || ! grep -Fq 'spec-verify-check --check failed' scripts/lint-fixtures.sh \
3912
- || ! grep -Fq 'python3 "$SPEC_VERIFY_CHECK" --check-expected "$d/expected.json"' scripts/lint-fixtures.sh \
3913
- || ! grep -Fq 'spec-verify-check --check-expected failed' scripts/lint-fixtures.sh \
3914
- || ! grep -Fq 'REJECTED_REGISTRY="${DEVLYN_REJECTED_FIXTURE_REGISTRY:-$REPO_ROOT/benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh}"' scripts/lint-fixtures.sh \
3915
- || ! grep -Fq 'declare -F rejected_pair_fixture_reason' scripts/lint-fixtures.sh \
3916
- || ! grep -Fq 'rejected fixture registry must define rejected_pair_fixture_reason' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
3917
- || ! grep -Fq 'malformed-rejected.sh' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
3918
- || ! grep -Fq 'NOTES.md records pair-candidate rejection but pair-rejected-fixtures.sh has no rejected reason' scripts/lint-fixtures.sh \
3919
- || ! grep -Fq 'test-active-headroom' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
3920
- || ! grep -Fq 'active-calibration-rejected-missing.out' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
3921
- || ! grep -Fq 'Rejected controls should remain replayable' benchmark/auto-resolve/fixtures/SCHEMA.md \
3922
- || ! grep -Fq 'pair-rejected-fixtures.sh' benchmark/auto-resolve/fixtures/SCHEMA.md \
3923
- || ! grep -Fq 'target.relative_to(fixture_root)' scripts/lint-fixtures.sh \
3924
- || ! grep -Fq '$BENCH_FIXTURE_DIR/...' benchmark/auto-resolve/fixtures/SCHEMA.md \
3925
- || ! grep -Fq 'must not escape the fixture directory' benchmark/auto-resolve/fixtures/SCHEMA.md \
3926
- || ! grep -Fq 'cd "$BENCH_FIXTURE_DIR"` indirection' benchmark/auto-resolve/fixtures/SCHEMA.md \
3927
- || ! grep -Fq 'success sentinel' benchmark/auto-resolve/fixtures/SCHEMA.md \
3928
- || ! grep -Fq 'def fallback_validate():' scripts/lint-fixtures.sh \
3929
- || ! grep -Fq 'expected.json must be an object' scripts/lint-fixtures.sh \
3930
- || ! grep -Fq 'schema_ok=0' scripts/lint-fixtures.sh \
3931
- || ! grep -Fq 'DEVLYN_LINT_FIXTURES_NO_JSONSCHEMA' scripts/lint-fixtures.sh \
3932
- || ! grep -Fq 'conditional risk-probe triggers' benchmark/auto-resolve/fixtures/SCHEMA.md; then
3933
- offenders="${offenders}"$'\n'"fixture lint must require high-risk fixtures to include resolve pair/risk trigger terms"
3934
- fi
3935
- unsafe_json_parser_refs=$(python3 - <<'PY'
3936
- import pathlib
3937
- import re
3938
-
3939
- root = pathlib.Path("benchmark/auto-resolve/scripts")
3940
- patterns = [
3941
- re.compile(r"json\.load\(open\("),
3942
- re.compile(r"json\.load\(response\)"),
3943
- re.compile(r"json\.loads\([^,\n]*(?:read_text\(\)|\bline\b|\bln\b)[^,\n]*\)"),
3944
- ]
3945
- for path in sorted(root.glob("*")):
3946
- if path.name.startswith("test-") or path.suffix not in {".py", ".sh"}:
3947
- continue
3948
- text = path.read_text(errors="ignore")
3949
- for line_no, line in enumerate(text.splitlines(), 1):
3950
- if any(pattern.search(line) for pattern in patterns):
3951
- print(f"{path}:{line_no}:{line.strip()}")
3952
- PY
3953
- )
3954
- if [ -n "$unsafe_json_parser_refs" ]; then
3955
- while IFS= read -r f; do
3956
- offenders="${offenders}"$'\n'"benchmark script uses unsafe JSON parser without strict constants: $f"
3957
- done <<< "$unsafe_json_parser_refs"
3958
- fi
3959
- if [ -z "$offenders" ]; then
3960
- ok "benchmark docs use current bare / solo_claude / pair-arm topology"
3961
- else
3962
- while IFS= read -r f; do bad "$f"; done <<< "$offenders"
3963
- fi
3964
-
3965
- # ---------------------------------------------------------------------------
3966
- # 9. Engine availability fails closed; stale silent-downgrade wording is forbidden.
3967
- # ---------------------------------------------------------------------------
3968
- section "Check 9: Engine availability fails closed"
3969
- offenders=$(grep -RInE 'codex-ping failed|codex-ping fail|engine downgraded: codex-unavailable|downgrades to Claude-only|silently downgrades|silently downgrade|silently switch to Claude|Codex CLI availability downgrade' \
3970
- config/skills CLAUDE.md README.md bin/ benchmark/auto-resolve/run-real-benchmark.md 2>/dev/null \
3971
- | grep -v 'roadmap-archival-workspace/' \
3972
- | grep -v 'devlyn:auto-resolve-workspace/' \
3973
- | grep -v 'devlyn:ideate-workspace/' \
3974
- | grep -v 'preflight-workspace/' \
3975
- || true)
3976
- if [ -z "$offenders" ]; then
3977
- ok "engine availability fail-closed wording canonical"
3978
- else
3979
- while IFS= read -r f; do bad "$f"; done <<< "$offenders"
3980
- fi
3981
-
3982
- # ---------------------------------------------------------------------------
3983
- # 9b. Release tag and package version parity.
3984
- # v2.2.3 was tagged while package.json still said 2.2.2. A mismatched npm
3985
- # package makes Codex/Claude installed-skill drift hard to diagnose because
3986
- # users can be on a tag whose package metadata points at a different build.
3987
- # ---------------------------------------------------------------------------
3988
- section "Check 9b: package.json version matches exact release tag"
3989
- exact_tag=$(git describe --tags --exact-match HEAD 2>/dev/null || true)
3990
- pkg_version=$(node -p "require('./package.json').version" 2>/dev/null || true)
3991
- if [ -z "$exact_tag" ]; then
3992
- ok "HEAD is not an exact tag — package/tag parity not applicable"
3993
- elif [ "$pkg_version" = "${exact_tag#v}" ]; then
3994
- ok "package.json version matches $exact_tag"
3995
- else
3996
- bad "package.json version '$pkg_version' does not match HEAD tag '$exact_tag'"
3997
- fi
3998
-
3999
- # ---------------------------------------------------------------------------
4000
- # (Check 7 retired iter-0034 Phase 4 cutover: the 4 findings-producing
4001
- # standalones — evaluate / review / clean / team-review — were deleted; the
4002
- # JSONL sidecar contract no longer has a surface to enforce.)
4003
- # ---------------------------------------------------------------------------
4004
- # 12. CLAUDE.md ↔ _shared/runtime-principles.md per-section excerpt parity (iter-0019.A).
4005
- # Sub-agent prompts inline the runtime contract from runtime-principles.md; that file
4006
- # must mirror the corresponding CLAUDE.md sections. Drift in one source-of-truth without
4007
- # the other produces silent behavioral divergence between session-level and sub-agent
4008
- # enforcement. Per-section markers `<!-- runtime-principles:section=NAME:begin/end -->`
4009
- # wrap each of the 3 mirrored sections (subtractive-first, goal-locked, evidence)
4010
- # in BOTH files. No-workaround discipline lives in CLAUDE.md Core Principle #1
4011
- # only and is not mirrored to runtime-principles.md.
4012
- # Check 12 extracts each named block from both files and diffs.
4013
- # ---------------------------------------------------------------------------
4014
- section "Check 12: CLAUDE.md ↔ runtime-principles.md per-section excerpt parity"
4015
- rp_src="config/skills/_shared/runtime-principles.md"
4016
- claude_src="CLAUDE.md"
4017
- rp_drift=0
4018
- expected_sections="subtractive-first goal-locked evidence"
4019
-
4020
- if [ ! -f "$rp_src" ]; then
4021
- bad "$rp_src — missing"
4022
- rp_drift=1
4023
- elif [ ! -f "$claude_src" ]; then
4024
- bad "$claude_src — missing"
4025
- rp_drift=1
4026
- else
4027
- # Topology: each marker appears exactly once per file.
4028
- for name in $expected_sections; do
4029
- for kind in begin end; do
4030
- marker="<!-- runtime-principles:section=${name}:${kind} -->"
4031
- for f in "$rp_src" "$claude_src"; do
4032
- count=$(grep -Fxc "$marker" "$f" 2>/dev/null || echo 0)
4033
- if [ "$count" -ne 1 ]; then
4034
- bad "${f}: marker '${marker}' appears ${count} times (expected 1)"
4035
- rp_drift=1
4036
- fi
4037
- done
4038
- done
4039
- done
4040
-
4041
- # Topology: in runtime-principles.md, all 4 sections must sit INSIDE the
4042
- # outer `:contract:` block AND appear in the canonical order. CLAUDE.md
4043
- # placement is free (sections may live in any order, anywhere in the file).
4044
- contract_begin_line=$(grep -Fxn '<!-- runtime-principles:contract:begin -->' "$rp_src" | head -1 | cut -d: -f1)
4045
- contract_end_line=$(grep -Fxn '<!-- runtime-principles:contract:end -->' "$rp_src" | head -1 | cut -d: -f1)
4046
- if [ -z "$contract_begin_line" ] || [ -z "$contract_end_line" ]; then
4047
- bad "${rp_src}: outer ':contract:begin/end' markers missing"
4048
- rp_drift=1
4049
- else
4050
- prev_line=0
4051
- for name in $expected_sections; do
4052
- sec_begin_line=$(grep -Fxn "<!-- runtime-principles:section=${name}:begin -->" "$rp_src" | head -1 | cut -d: -f1)
4053
- sec_end_line=$(grep -Fxn "<!-- runtime-principles:section=${name}:end -->" "$rp_src" | head -1 | cut -d: -f1)
4054
- if [ -n "$sec_begin_line" ] && [ -n "$sec_end_line" ]; then
4055
- if [ "$sec_begin_line" -le "$contract_begin_line" ] || [ "$sec_end_line" -ge "$contract_end_line" ]; then
4056
- bad "${rp_src}: section '${name}' is outside the ':contract:' block"
4057
- rp_drift=1
4058
- fi
4059
- if [ "$sec_begin_line" -lt "$prev_line" ]; then
4060
- bad "${rp_src}: section '${name}' is out of canonical order (expected: ${expected_sections})"
4061
- rp_drift=1
4062
- fi
4063
- prev_line=$sec_end_line
4064
- fi
4065
- done
4066
- fi
4067
-
4068
- # Content: byte-compare each section block via diff over temp files.
4069
- # awk-into-tmpfile preserves trailing newlines (command substitution strips them).
4070
- if make_temp_file tmp_rp && make_temp_file tmp_claude; then
4071
- for name in $expected_sections; do
4072
- begin="<!-- runtime-principles:section=${name}:begin -->"
4073
- end="<!-- runtime-principles:section=${name}:end -->"
4074
- awk -v b="$begin" -v e="$end" '$0==b{f=1;next}$0==e{f=0}f' "$rp_src" > "$tmp_rp"
4075
- awk -v b="$begin" -v e="$end" '$0==b{f=1;next}$0==e{f=0}f' "$claude_src" > "$tmp_claude"
4076
- if [ ! -s "$tmp_rp" ]; then
4077
- bad "${name}: empty/missing block in $rp_src"
4078
- rp_drift=1
4079
- continue
4080
- fi
4081
- if [ ! -s "$tmp_claude" ]; then
4082
- bad "${name}: empty/missing block in $claude_src"
4083
- rp_drift=1
4084
- continue
4085
- fi
4086
- if ! diff -q "$tmp_rp" "$tmp_claude" >/dev/null 2>&1; then
4087
- bad "${name}: CLAUDE.md and runtime-principles.md content differ"
4088
- rp_drift=1
4089
- fi
4090
- done
4091
- rm -f "$tmp_rp" "$tmp_claude"
4092
- else
4093
- rp_drift=1
4094
- fi
4095
-
4096
- if [ $rp_drift -eq 0 ]; then
4097
- ok "all 3 mirrored contract sections in parity (subtractive-first / goal-locked / evidence) — markers, topology, content"
4098
- fi
4099
- fi
4100
-
4101
- # ---------------------------------------------------------------------------
4102
- # 13. pair-plan idgen output is deterministic across consecutive runs (iter-0022).
4103
- # Same input → byte-identical canonical_id_registry.json. Catches accidental
4104
- # dict-order, float-printing, or timestamp-leak regressions in idgen.
4105
- # Runs twice on F2 with --generated-at pinned and compares sha256.
4106
- # ---------------------------------------------------------------------------
4107
- section "Check 13: pair-plan-idgen.py output deterministic across runs (F2)"
4108
- idgen="benchmark/auto-resolve/scripts/pair-plan-idgen.py"
4109
- fixture="benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand"
4110
- if [ ! -x "$idgen" ] && [ ! -f "$idgen" ]; then
4111
- bad "Check 13 prerequisite missing: $idgen"
4112
- elif [ ! -d "$fixture" ]; then
4113
- bad "Check 13 prerequisite missing: $fixture"
4114
- else
4115
- if make_temp_file tmp1 && make_temp_file tmp2; then
4116
- if python3 "$idgen" --fixture "$fixture" --generated-at 2026-04-29T18:30:00Z --output "$tmp1" >/dev/null 2>&1 \
4117
- && python3 "$idgen" --fixture "$fixture" --generated-at 2026-04-29T18:30:00Z --output "$tmp2" >/dev/null 2>&1; then
4118
- sha1=$(shasum -a 256 "$tmp1" | awk '{print $1}')
4119
- sha2=$(shasum -a 256 "$tmp2" | awk '{print $1}')
4120
- if [ "$sha1" = "$sha2" ]; then
4121
- ok "F2 registry sha256 stable across two idgen runs ($sha1)"
4122
- else
4123
- bad "F2 registry sha256 drift: run1=$sha1 run2=$sha2"
4124
- fi
4125
- else
4126
- bad "idgen invocation failed; cannot verify determinism"
4127
- fi
4128
- else
4129
- bad "idgen temp files unavailable; cannot verify determinism"
4130
- fi
4131
- if make_temp_dir tmp_bad && make_temp_file idgen_nan_out /tmp/pair-plan-idgen-nan.out.XXXXXX; then
4132
- cp -R "$fixture/." "$tmp_bad/"
4133
- printf '{"verification_commands": NaN}\n' > "$tmp_bad/expected.json"
4134
- if python3 "$idgen" --fixture "$tmp_bad" --generated-at 2026-04-29T18:30:00Z >"$idgen_nan_out" 2>&1; then
4135
- bad "pair-plan-idgen.py accepted NaN in expected.json"
4136
- elif ! grep -Fq 'invalid JSON numeric constant: NaN' "$idgen_nan_out"; then
4137
- bad "pair-plan-idgen.py NaN failure did not cite invalid JSON numeric constant"
4138
- fi
4139
- rm -rf "$tmp_bad" "$idgen_nan_out"
4140
- fi
4141
-
4142
- if make_temp_file bad_plan && make_temp_file plan_lint_nan_out /tmp/pair-plan-lint-nan.out.XXXXXX; then
4143
- printf '{"schema_version": NaN}\n' > "$bad_plan"
4144
- if python3 benchmark/auto-resolve/scripts/pair-plan-lint.py --plan "$bad_plan" --quiet >"$plan_lint_nan_out" 2>&1; then
4145
- bad "pair-plan-lint.py accepted NaN in pair-plan.json"
4146
- elif ! grep -Fq '"code": "plan_invalid_json"' "$plan_lint_nan_out" \
4147
- || ! grep -Fq 'invalid JSON numeric constant: NaN' "$plan_lint_nan_out"; then
4148
- bad "pair-plan-lint.py NaN failure did not report plan_invalid_json with invalid numeric constant"
4149
- fi
4150
- rm -f "$bad_plan" "$plan_lint_nan_out"
4151
- fi
4152
- rm -f "${tmp1:-}" "${tmp2:-}"
4153
- fi
4154
-
4155
- # ---------------------------------------------------------------------------
4156
- # 14. F9 fixture id matches the shipped 2-skill contract (iter-0033a, 2026-04-30).
4157
- # `/devlyn:preflight` was folded into `/devlyn:resolve`'s VERIFY phase; the
4158
- # legacy F9 dir name (`F9-e2e-ideate-to-preflight`) is misleading once
4159
- # preflight is gone. The retired copy lives under `fixtures/retired/` for
4160
- # replay; the live fixture must be `F9-e2e-ideate-to-resolve`. Any other
4161
- # non-retired reference to the old id is a stale rename.
4162
- # ---------------------------------------------------------------------------
4163
- section "Check 14: F9 fixture id matches 2-skill contract"
4164
- f9_drift=0
4165
- if [ ! -d "benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve" ]; then
4166
- bad "live F9 fixture missing at benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve"
4167
- f9_drift=1
4168
- fi
4169
- # Stale references outside fixtures/retired/ are bugs. Examine line content
4170
- # (not just filename) so files that legitimately mention the retired *path*
4171
- # (e.g. fixtures/F9-e2e-ideate-to-resolve/NOTES.md explaining where the OLD
4172
- # version lives) pass while genuine stale references fail. Excluded scopes:
4173
- # benchmark/auto-resolve/results/ (historical run artifacts, frozen) and
4174
- # scripts/lint-skills.sh itself (carries the pattern in this check).
4175
- stale=$(git grep -In -- 'F9-e2e-ideate-to-preflight' -- \
4176
- config/skills \
4177
- benchmark \
4178
- scripts \
4179
- CLAUDE.md \
4180
- README.md \
4181
- ':!benchmark/auto-resolve/results/**' \
4182
- 2>/dev/null \
4183
- | grep -v '^benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/' \
4184
- | grep -v '^scripts/lint-skills\.sh:' \
4185
- | grep -v 'fixtures/retired/F9-e2e-ideate-to-preflight' \
4186
- || true)
4187
- if [ -n "$stale" ]; then
4188
- while IFS= read -r f; do bad "stale F9-e2e-ideate-to-preflight reference: $f"; done <<< "$stale"
4189
- f9_drift=1
4190
- fi
4191
- if [ $f9_drift -eq 0 ]; then
4192
- ok "F9 fixture id is canonical (F9-e2e-ideate-to-resolve); no stale refs outside retired/"
4193
- fi
4194
-
4195
- # ---------------------------------------------------------------------------
4196
- # 15. Current pair-evidence fixtures carry their local evidence handoff notes.
4197
- # The audit artifacts are canonical, but fixture-level notes keep future
4198
- # fixture edits from losing why a candidate currently counts as solo<pair
4199
- # evidence.
4200
- # ---------------------------------------------------------------------------
4201
- section "Check 15: Pair evidence fixture notes cite current passing runs"
4202
- if python3 - <<'PY'
4203
- import importlib.util
4204
- import pathlib
4205
- import sys
4206
-
4207
- script = pathlib.Path("benchmark/auto-resolve/scripts/pair-candidate-frontier.py")
4208
- spec = importlib.util.spec_from_file_location("pair_candidate_frontier", script)
4209
- module = importlib.util.module_from_spec(spec)
4210
- assert spec.loader is not None
4211
- spec.loader.exec_module(module)
4212
-
4213
- report = module.build_report(
4214
- fixtures_root=pathlib.Path("benchmark/auto-resolve/fixtures"),
4215
- registry=pathlib.Path("benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh"),
4216
- results_root=pathlib.Path("benchmark/auto-resolve/results"),
4217
- )
4218
- errors = []
4219
- for row in report["rows"]:
4220
- if row.get("status") != "pair_evidence_passed":
4221
- continue
4222
- fixture = row["fixture"]
4223
- best = module.best_pair_evidence(row.get("passing_pair_evidence", []))
4224
- if best is None:
4225
- errors.append(f"{fixture}: missing complete pair evidence")
4226
- continue
4227
- notes_path = pathlib.Path("benchmark/auto-resolve/fixtures") / fixture / "NOTES.md"
4228
- try:
4229
- notes = notes_path.read_text(encoding="utf8")
4230
- except OSError:
4231
- errors.append(f"{fixture}: NOTES.md missing")
4232
- continue
4233
- required = [
4234
- str(best["run_id"]),
4235
- "pair_evidence_passed",
4236
- f"bare `{best['bare_score']}`",
4237
- f"solo_claude `{best['solo_score']}`",
4238
- f"pair `{best['pair_score']}`",
4239
- f"margin `{best['pair_margin']:+d}`",
4240
- f"wall `{best['pair_solo_wall_ratio']:.2f}x`",
4241
- f"arm `{best['pair_arm']}`",
4242
- ]
4243
- missing = [item for item in required if item not in notes]
4244
- if missing:
4245
- errors.append(f"{fixture}: NOTES.md missing {', '.join(missing)}")
4246
- if errors:
4247
- raise SystemExit("\n".join(errors))
4248
- PY
4249
- then
4250
- ok "current pair-evidence fixture notes cite passing run ids"
4251
- else
4252
- bad "current pair-evidence fixture NOTES.md files must cite passing run ids and pair_evidence_passed"
4253
- fi
4254
-
4255
- # ---------------------------------------------------------------------------
4256
- # Summary.
4257
- # ---------------------------------------------------------------------------
4258
- echo
4259
- if [ $fail -eq 0 ]; then
4260
- printf '%sAll checks passed.%s\n' "$green" "$reset"
4261
- exit 0
4262
- else
4263
- printf '%sLint failed.%s Fix the offenders above.\n' "$red" "$reset"
4264
- exit 1
4265
- fi