devlyn-cli 3.2.1 → 3.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +2 -2
- package/CLAUDE.md +2 -2
- package/README.md +12 -102
- package/bin/devlyn.js +13 -225
- package/bin/instruction-templates.json +8 -0
- package/config/skills/_shared/adapters/grok.md +1 -2
- package/config/skills/_shared/archive_run.py +15 -3
- package/config/skills/_shared/judge-role-evidence.py +21 -1
- package/config/skills/_shared/runtime-principles.md +1 -1
- package/config/skills/_shared/spec-verify-check.py +0 -653
- package/config/skills/_shared/state-phase-write.py +351 -25
- package/config/skills/_shared/task-complete.py +660 -150
- package/config/skills/_shared/terminal-claim-check.py +229 -4
- package/config/skills/_shared/verify-merge-findings.py +93 -2
- package/config/skills/devlyn:ideate/SKILL.md +3 -4
- package/config/skills/devlyn:ideate/references/elicitation.md +2 -35
- package/config/skills/devlyn:ideate/references/from-spec-mode.md +3 -18
- package/config/skills/devlyn:ideate/references/project-mode.md +1 -16
- package/config/skills/devlyn:ideate/references/spec-template.md +0 -2
- package/config/skills/devlyn:queue/SKILL.md +6 -4
- package/config/skills/devlyn:resolve/SKILL.md +16 -25
- package/config/skills/devlyn:resolve/references/free-form-mode.md +0 -14
- package/config/skills/devlyn:resolve/references/outer-loop.md +1 -1
- package/config/skills/devlyn:resolve/references/phases/build-gate.md +1 -1
- package/config/skills/devlyn:resolve/references/phases/cleanup.md +3 -3
- package/config/skills/devlyn:resolve/references/phases/implement.md +1 -1
- package/config/skills/devlyn:resolve/references/phases/probe-derive.md +1 -16
- package/config/skills/devlyn:resolve/references/phases/verify.md +4 -9
- package/config/skills/devlyn:resolve/references/state-schema.md +9 -7
- package/config/skills/devlyn:resolve/references/task-completion.md +47 -38
- package/{config/skills → optional-skills}/code-health-standards/SKILL.md +4 -13
- package/{config/skills → optional-skills}/code-review-standards/SKILL.md +4 -12
- package/{config/skills → optional-skills}/root-cause-analysis/SKILL.md +4 -12
- package/{config/skills → optional-skills}/ui-implementation-standards/SKILL.md +4 -14
- package/package.json +1 -57
- package/benchmark/auto-resolve/BENCHMARK-DESIGN.md +0 -289
- package/benchmark/auto-resolve/BENCHMARK-RESULTS.md +0 -341
- package/benchmark/auto-resolve/README.md +0 -696
- package/benchmark/auto-resolve/RUBRIC.md +0 -177
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/NOTES.md +0 -34
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/expected.json +0 -68
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/setup.sh +0 -4
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/spec.md +0 -44
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/task.ko.txt +0 -8
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/task.txt +0 -8
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/NOTES.md +0 -68
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/expected.json +0 -60
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/setup.sh +0 -17
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/spec.md +0 -51
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/verifiers/invalid.js +0 -29
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/verifiers/parallel.js +0 -50
- package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/NOTES.md +0 -75
- package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/expected.json +0 -52
- package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/setup.sh +0 -171
- package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/spec.md +0 -50
- package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/NOTES.md +0 -89
- package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/expected.json +0 -74
- package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/setup.sh +0 -251
- package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/spec.md +0 -57
- package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/task.txt +0 -13
- package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/verifiers/replay-malformed-body.js +0 -64
- package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/NOTES.md +0 -104
- package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/expected.json +0 -46
- package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/setup.sh +0 -336
- package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/spec.md +0 -51
- package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/NOTES.md +0 -38
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/expected.json +0 -64
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/setup.sh +0 -32
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/spec.md +0 -63
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/verifiers/exact-success.js +0 -54
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/verifiers/no-hardcoded-pricing.js +0 -47
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/verifiers/stock-error.js +0 -45
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/NOTES.md +0 -57
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/expected-pair-plan-registry.json +0 -170
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/expected.json +0 -84
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/metadata.json +0 -21
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/pair-plan.sample-fail.json +0 -214
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/pair-plan.sample-pass.json +0 -223
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/setup.sh +0 -5
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/spec.md +0 -55
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/task.txt +0 -14
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/NOTES.md +0 -39
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/expected.json +0 -62
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/setup.sh +0 -2
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/spec.md +0 -67
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/verifiers/error-order.js +0 -55
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/verifiers/priority-blocked.js +0 -48
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/NOTES.md +0 -35
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/expected.json +0 -56
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/setup.sh +0 -2
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/spec.md +0 -64
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/verifiers/conflicting-duplicate.js +0 -34
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/verifiers/idempotent-close.js +0 -41
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/NOTES.md +0 -39
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/expected.json +0 -56
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/setup.sh +0 -2
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/spec.md +0 -76
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/verifiers/priority-rollback.js +0 -64
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/verifiers/single-warehouse-fefo.js +0 -66
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/NOTES.md +0 -40
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/expected.json +0 -66
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/setup.sh +0 -36
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/spec.md +0 -71
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/verifiers/catalog-source.js +0 -57
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/verifiers/exact-success.js +0 -63
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/verifiers/stock-error.js +0 -34
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/NOTES.md +0 -31
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/expected.json +0 -68
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/setup.sh +0 -17
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/spec.md +0 -68
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/verifiers/conflicting-duplicate.js +0 -29
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/verifiers/exact-payout.js +0 -58
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/verifiers/rules-source.js +0 -56
- package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/NOTES.md +0 -35
- package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/expected-pair-plan-registry.json +0 -162
- package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/expected.json +0 -67
- package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/metadata.json +0 -19
- package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/setup.sh +0 -4
- package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/spec.md +0 -55
- package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/NOTES.md +0 -34
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/expected.json +0 -57
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/setup.sh +0 -2
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/spec.md +0 -67
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/verifiers/duplicate-event-error.js +0 -35
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/verifiers/priority-transfer-rollback.js +0 -53
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/NOTES.md +0 -38
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/expected.json +0 -57
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/setup.sh +0 -2
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/spec.md +0 -70
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/task.txt +0 -3
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/verifiers/duplicate-renewal-error.js +0 -42
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/verifiers/priority-credit-rollback.js +0 -70
- package/benchmark/auto-resolve/fixtures/F4-web-browser-design/NOTES.md +0 -47
- package/benchmark/auto-resolve/fixtures/F4-web-browser-design/expected.json +0 -57
- package/benchmark/auto-resolve/fixtures/F4-web-browser-design/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F4-web-browser-design/setup.sh +0 -6
- package/benchmark/auto-resolve/fixtures/F4-web-browser-design/spec.md +0 -48
- package/benchmark/auto-resolve/fixtures/F4-web-browser-design/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/NOTES.md +0 -45
- package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/expected.json +0 -65
- package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/setup.sh +0 -55
- package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/spec.md +0 -48
- package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/NOTES.md +0 -43
- package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/expected.json +0 -77
- package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/setup.sh +0 -4
- package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/spec.md +0 -48
- package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/task.txt +0 -10
- package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/NOTES.md +0 -57
- package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/expected.json +0 -76
- package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/setup.sh +0 -36
- package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/spec.md +0 -45
- package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/NOTES.md +0 -53
- package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/expected.json +0 -63
- package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/setup.sh +0 -4
- package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/spec.md +0 -48
- package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/task.txt +0 -1
- package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/NOTES.md +0 -105
- package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/expected.json +0 -74
- package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/setup.sh +0 -28
- package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/spec.md +0 -59
- package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/task.txt +0 -5
- package/benchmark/auto-resolve/fixtures/SCHEMA.md +0 -188
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/NOTES.md +0 -37
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/RETIRED.md +0 -13
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/expected.json +0 -56
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/setup.sh +0 -18
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/spec.md +0 -69
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/verifiers/exact-proration.js +0 -48
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/verifiers/rules-source-and-conflict.js +0 -79
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/NOTES.md +0 -54
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/RETIRED.md +0 -7
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/expected.json +0 -67
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/setup.sh +0 -2
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/spec.md +0 -67
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/task.txt +0 -5
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/verifiers/policy-precedence.js +0 -72
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/verifiers/validation-and-immutability.js +0 -43
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/verifiers/validation-boundary.js +0 -116
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/NOTES.md +0 -35
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/RETIRED.md +0 -12
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/expected.json +0 -58
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/setup.sh +0 -2
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/spec.md +0 -73
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/task.txt +0 -17
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/verifiers/mixed-idempotent-settlement.js +0 -53
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/verifiers/rejection-boundaries.js +0 -74
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/NOTES.md +0 -28
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/RETIRED.md +0 -15
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/expected.json +0 -57
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/setup.sh +0 -50
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/spec.md +0 -59
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/verifiers/ordered-settlement.js +0 -88
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/verifiers/reject-invalid.js +0 -45
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/NOTES.md +0 -30
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/RETIRED.md +0 -16
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/expected.json +0 -64
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/setup.sh +0 -21
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/spec.md +0 -58
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/verifiers/dup-op.js +0 -54
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/verifiers/priority-apply.js +0 -82
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/verifiers/rollback.js +0 -77
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/NOTES.md +0 -99
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/RETIRED.md +0 -18
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/expected.json +0 -64
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/setup.sh +0 -4
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/spec.md +0 -61
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/verifiers/correctness-small.js +0 -135
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/verifiers/scale.js +0 -100
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/verifiers/validation.js +0 -75
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/NOTES.md +0 -93
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/RETIRED.md +0 -16
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/expected.json +0 -65
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/setup.sh +0 -18
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/spec.md +0 -66
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/verifiers/correctness-small.js +0 -84
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/verifiers/scale.js +0 -184
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/verifiers/validation.js +0 -74
- package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/NOTES.md +0 -60
- package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/RETIRED.md +0 -29
- package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/expected.json +0 -73
- package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/setup.sh +0 -28
- package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/spec.md +0 -58
- package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/task.txt +0 -5
- package/benchmark/auto-resolve/fixtures/test-repo/README.md +0 -12
- package/benchmark/auto-resolve/fixtures/test-repo/bin/cli.js +0 -63
- package/benchmark/auto-resolve/fixtures/test-repo/package-lock.json +0 -823
- package/benchmark/auto-resolve/fixtures/test-repo/package.json +0 -22
- package/benchmark/auto-resolve/fixtures/test-repo/playwright.config.js +0 -17
- package/benchmark/auto-resolve/fixtures/test-repo/server/index.js +0 -37
- package/benchmark/auto-resolve/fixtures/test-repo/tests/cli.test.js +0 -25
- package/benchmark/auto-resolve/fixtures/test-repo/tests/server.test.js +0 -58
- package/benchmark/auto-resolve/fixtures/test-repo/web/index.html +0 -37
- package/benchmark/auto-resolve/results/20260507-f10-f11-tier1-full-pipeline/headroom-gate.json +0 -25
- package/benchmark/auto-resolve/results/20260507-f10-f11-tier1-full-pipeline/headroom-gate.md +0 -10
- package/benchmark/auto-resolve/results/20260508-f22-exact-error-headroom/headroom-gate.json +0 -18
- package/benchmark/auto-resolve/results/20260508-f22-exact-error-headroom/headroom-gate.md +0 -9
- package/benchmark/auto-resolve/results/20260508-f26-headroom/headroom-gate.json +0 -18
- package/benchmark/auto-resolve/results/20260508-f26-headroom/headroom-gate.md +0 -9
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F16-cli-quote-tax-rules/l2_risk_probes/result.json +0 -28
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F23-cli-fulfillment-wave/l2_risk_probes/result.json +0 -28
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F25-cli-cart-promotion-rules/l2_risk_probes/result.json +0 -30
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.json +0 -82
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.md +0 -18
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.json +0 -46
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.md +0 -17
- package/benchmark/auto-resolve/results/20260511-f12-webhook-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260511-f12-webhook-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260511-f15-concurrency-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260511-f15-concurrency-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/F21-cli-scheduler-priority/l2_risk_probes/result.json +0 -32
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.json +0 -44
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.md +0 -16
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.json +0 -18
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.md +0 -9
- package/benchmark/auto-resolve/results/20260511-f3-http-error-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260511-f3-http-error-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f2-medium-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f2-medium-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f31-seat-rebalance-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f31-seat-rebalance-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f32-subscription-renewal-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f32-subscription-renewal-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f4-web-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f4-web-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f5-fixloop-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f5-fixloop-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f6-checksum-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f6-checksum-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f7-scope-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f7-scope-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f9-e2e-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f9-e2e-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/run-real-benchmark.md +0 -303
- package/benchmark/auto-resolve/scripts/audit-headroom-rejections.py +0 -441
- package/benchmark/auto-resolve/scripts/audit-pair-evidence.py +0 -1256
- package/benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py +0 -306
- package/benchmark/auto-resolve/scripts/check-f9-artifacts.py +0 -268
- package/benchmark/auto-resolve/scripts/collect-swebench-predictions.py +0 -108
- package/benchmark/auto-resolve/scripts/compile-report.py +0 -493
- package/benchmark/auto-resolve/scripts/fetch-swebench-instances.py +0 -129
- package/benchmark/auto-resolve/scripts/frozen-verify-gate.py +0 -434
- package/benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py +0 -612
- package/benchmark/auto-resolve/scripts/headroom-gate.py +0 -378
- package/benchmark/auto-resolve/scripts/iter-0033c-compare.py +0 -683
- package/benchmark/auto-resolve/scripts/iter-0033c-l1-summary.py +0 -97
- package/benchmark/auto-resolve/scripts/judge-opus-pass.sh +0 -542
- package/benchmark/auto-resolve/scripts/judge.sh +0 -565
- package/benchmark/auto-resolve/scripts/oracle-scope-tier-a.py +0 -256
- package/benchmark/auto-resolve/scripts/oracle-scope-tier-b.py +0 -287
- package/benchmark/auto-resolve/scripts/oracle-test-fidelity.py +0 -328
- package/benchmark/auto-resolve/scripts/pair-candidate-frontier.py +0 -469
- package/benchmark/auto-resolve/scripts/pair-plan-idgen.py +0 -401
- package/benchmark/auto-resolve/scripts/pair-plan-lint.py +0 -475
- package/benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh +0 -103
- package/benchmark/auto-resolve/scripts/pair_evidence_contract.py +0 -270
- package/benchmark/auto-resolve/scripts/prepare-swebench-frozen-case.py +0 -273
- package/benchmark/auto-resolve/scripts/prepare-swebench-frozen-corpus.py +0 -148
- package/benchmark/auto-resolve/scripts/prepare-swebench-solver-worktree.py +0 -210
- package/benchmark/auto-resolve/scripts/recent-benchmark-summary.py +0 -232
- package/benchmark/auto-resolve/scripts/run-fixture.sh +0 -1024
- package/benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh +0 -683
- package/benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh +0 -458
- package/benchmark/auto-resolve/scripts/run-headroom-candidate.sh +0 -336
- package/benchmark/auto-resolve/scripts/run-iter-0033c.sh +0 -208
- package/benchmark/auto-resolve/scripts/run-suite.sh +0 -251
- package/benchmark/auto-resolve/scripts/run-swebench-frozen-corpus.sh +0 -310
- package/benchmark/auto-resolve/scripts/run-swebench-solver-batch.sh +0 -257
- package/benchmark/auto-resolve/scripts/ship-gate.py +0 -391
- package/benchmark/auto-resolve/scripts/solo-ceiling-avoidance.py +0 -53
- package/benchmark/auto-resolve/scripts/solo-headroom-hypothesis.py +0 -77
- package/benchmark/auto-resolve/scripts/swebench-frozen-matrix.py +0 -478
- package/benchmark/auto-resolve/scripts/test-audit-headroom-rejections.sh +0 -288
- package/benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh +0 -1672
- package/benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh +0 -1015
- package/benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh +0 -491
- package/benchmark/auto-resolve/scripts/test-check-f9-artifacts.sh +0 -91
- package/benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh +0 -517
- package/benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh +0 -610
- package/benchmark/auto-resolve/scripts/test-headroom-gate.sh +0 -401
- package/benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh +0 -525
- package/benchmark/auto-resolve/scripts/test-iter-0033c-l1-summary.sh +0 -254
- package/benchmark/auto-resolve/scripts/test-lint-fixtures.sh +0 -580
- package/benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh +0 -591
- package/benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh +0 -497
- package/benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh +0 -401
- package/benchmark/auto-resolve/scripts/test-run-swebench-solver-batch.sh +0 -111
- package/benchmark/auto-resolve/scripts/test-ship-gate.sh +0 -1189
- package/benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh +0 -1221
- package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/NOTES.md +0 -28
- package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/expected.json +0 -63
- package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/metadata.json +0 -10
- package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/setup.sh +0 -3
- package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/spec.md +0 -47
- package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/task.txt +0 -1
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/NOTES.md +0 -34
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/expected.json +0 -53
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/metadata.json +0 -10
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/setup.sh +0 -3
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/spec.md +0 -50
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/task.txt +0 -1
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/verifiers/duplicate-order-error.js +0 -27
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/verifiers/priority-stock-reservation.js +0 -44
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/NOTES.md +0 -34
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/expected.json +0 -55
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/metadata.json +0 -10
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/setup.sh +0 -3
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/spec.md +0 -52
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/task.txt +0 -1
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/verifiers/duplicate-ticket-error.js +0 -29
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/verifiers/priority-agent-assignment.js +0 -48
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/NOTES.md +0 -34
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/expected.json +0 -55
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/metadata.json +0 -10
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/setup.sh +0 -3
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/spec.md +0 -55
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/task.txt +0 -1
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/verifiers/duplicate-return-error.js +0 -43
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/verifiers/priority-return-routing.js +0 -70
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/NOTES.md +0 -37
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/expected.json +0 -54
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/metadata.json +0 -10
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/setup.sh +0 -3
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/spec.md +0 -59
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/task.txt +0 -1
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/verifiers/credit-ledger-priority.js +0 -98
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/verifiers/duplicate-charge-error.js +0 -38
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/NOTES.md +0 -36
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/expected.json +0 -56
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/metadata.json +0 -10
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/setup.sh +0 -3
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/spec.md +0 -59
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/task.txt +0 -1
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/verifiers/duplicate-refund-error.js +0 -41
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/verifiers/priority-refund-ledger.js +0 -65
- package/config/skills/_shared/pair-plan-schema.md +0 -298
- package/scripts/lint-fixtures.sh +0 -352
- package/scripts/lint-shadow-fixtures.sh +0 -58
- package/scripts/lint-skills.sh +0 -4265
- package/scripts/test-owner-phases.py +0 -238
|
@@ -1,59 +0,0 @@
|
|
|
1
|
-
---
|
|
2
|
-
id: "F34-cli-settle-inflight"
|
|
3
|
-
title: "Settle command with in-flight authorizations"
|
|
4
|
-
status: planned
|
|
5
|
-
complexity: high
|
|
6
|
-
depends-on: []
|
|
7
|
-
---
|
|
8
|
-
|
|
9
|
-
# F34 Settle command with in-flight authorizations
|
|
10
|
-
|
|
11
|
-
## Context
|
|
12
|
-
|
|
13
|
-
`bench-cli` gets a `settle` command that previews a batch of credit
|
|
14
|
-
transfers between accounts. Every transfer must be authorized through the
|
|
15
|
-
seeded gateway module before settlement. The gateway is slow, so
|
|
16
|
-
authorizations for different transfers must be able to overlap — but the
|
|
17
|
-
ledger math must stay exact and order-stable.
|
|
18
|
-
|
|
19
|
-
## Requirements
|
|
20
|
-
|
|
21
|
-
- [ ] `bench-cli settle --input <path>` reads JSON shaped as `{ "transfers": [{ "id": string, "from": string, "to": string, "amount": number }] }`.
|
|
22
|
-
- [ ] Account starting balances come from `data/accounts.json`. The command must not modify `data/accounts.json`.
|
|
23
|
-
- [ ] Every transfer is authorized by calling `authorize(transfer)` from `lib/gateway.js` exactly once per transfer. Do not modify, reimplement, or bypass `lib/gateway.js`.
|
|
24
|
-
- [ ] Authorizations must be able to overlap: while one transfer's authorization is awaited, later transfers' authorizations must be able to start. The seeded gateway records the maximum number of authorizations in flight at once in `data/gateway-stats.json`; settling the six-transfer sample in `## Verification` must record `high_water` of at least `3`.
|
|
25
|
-
- [ ] Regardless of the order in which authorizations complete, transfers settle in file order, each exactly once. A transfer is accepted when the sender's balance — after all earlier-in-file accepted transfers — covers the amount; otherwise it is rejected.
|
|
26
|
-
- [ ] Each rejected transfer produces exact row shape `{ "id": string, "error": "insufficient_funds", "available": number, "requested": number }`, where `available` is the sender's balance at that transfer's turn in file order.
|
|
27
|
-
- [ ] Validation happens before any authorization starts. Invalid JSON, missing or non-string `id`/`from`/`to`, unknown account, or non-positive or non-integer `amount` exits `2` and writes exactly one JSON error object to stderr with shape `{ "error": "invalid_input", "reason": string }`, printing nothing to stdout.
|
|
28
|
-
- [ ] On success, write exactly one JSON object to stdout and no stderr. Keys: `accepted` (array of transfer ids in file order), `rejected` (array of rejected rows in file order), `balances` (object with the final balance of every account in `data/accounts.json`).
|
|
29
|
-
- [ ] `tests/cli.test.js` is updated. Existing tests still pass AND at least two new tests cover `settle`: one where a later-in-file transfer depends on an earlier-in-file transfer's settlement, and one rejection.
|
|
30
|
-
|
|
31
|
-
## Constraints
|
|
32
|
-
|
|
33
|
-
- **No new npm dependencies.**
|
|
34
|
-
- **No silent catches.** If parsing or file reading fails, emit a visible JSON error to stderr and exit `2`.
|
|
35
|
-
- **No extra stdout/stderr text** on the success path; downstream tooling parses stdout as JSON.
|
|
36
|
-
- Balances are integer credits; no floating-point money.
|
|
37
|
-
|
|
38
|
-
## Out of Scope
|
|
39
|
-
|
|
40
|
-
- Persisting settlement results (settle is a preview; `data/accounts.json` stays untouched).
|
|
41
|
-
- Changing server routes, web UI, or `lib/gateway.js`.
|
|
42
|
-
- Retry, timeout, or failure handling for the gateway beyond what it returns.
|
|
43
|
-
- Touching `server/`, `web/`, or `tests/server.test.js`.
|
|
44
|
-
|
|
45
|
-
## Verification
|
|
46
|
-
|
|
47
|
-
- `node --test tests/cli.test.js` exits 0.
|
|
48
|
-
- Settling a six-transfer file where authorizations complete in reverse file order still yields file-order settlement: exact `accepted`, `rejected`, and `balances` values.
|
|
49
|
-
- `data/gateway-stats.json` shows `calls` equal to the number of transfers and `high_water` of at least `3`.
|
|
50
|
-
- An unknown account exits `2`, prints one JSON error to stderr, and prints no stdout.
|
|
51
|
-
- `git diff --stat` shows only `bin/cli.js` and `tests/cli.test.js` touched (accounts and gateway are seeded by setup, not the arm).
|
|
52
|
-
|
|
53
|
-
## Solo-headroom hypothesis
|
|
54
|
-
|
|
55
|
-
A capable solo_claude baseline is expected to miss the decoupling of
|
|
56
|
-
authorization completion order from file-order settlement — applying
|
|
57
|
-
balance changes as authorizations resolve instead of at each transfer's
|
|
58
|
-
file-order turn; observable command
|
|
59
|
-
`node "$BENCH_FIXTURE_DIR/verifiers/ordered-settlement.js"` exposes the miss.
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
Add a `settle` command to `bench-cli` so users can run `bench-cli settle --input <path>` with a transfers JSON file. Starting balances come from `data/accounts.json` (read-only — settle is a preview and must not modify it). Every transfer must be authorized by calling `authorize(transfer)` from the seeded `lib/gateway.js` exactly once; do not modify or bypass that module.
|
|
2
|
-
|
|
3
|
-
The gateway is slow, so authorizations for different transfers must be able to overlap — while one is awaited, later ones must be able to start. The seeded gateway records `calls` and the in-flight `high_water` mark in `data/gateway-stats.json`; a six-transfer batch must reach a high-water mark of at least 3.
|
|
4
|
-
|
|
5
|
-
No matter which order authorizations finish in, transfers settle in file order, each exactly once: a transfer is accepted when the sender's balance after all earlier-in-file accepted transfers covers the amount, otherwise it is rejected with the exact row `{ "id", "error": "insufficient_funds", "available", "requested" }` where `available` is the balance at that transfer's file-order turn.
|
|
6
|
-
|
|
7
|
-
Validate the whole file before any authorization starts: bad JSON, missing fields, unknown accounts, or non-positive/non-integer amounts exit `2` with exactly one JSON error object `{ "error": "invalid_input", "reason": string }` on stderr and nothing on stdout. On success print exactly one JSON object with `accepted` (ids in file order), `rejected` (rows in file order), and `balances` (every account) — and nothing on stderr.
|
|
8
|
-
|
|
9
|
-
Update `tests/cli.test.js`: existing tests still pass, plus at least two new settle tests (one where a later transfer depends on an earlier transfer's settlement, one rejection). No new npm dependencies. Only touch `bin/cli.js` and `tests/cli.test.js`.
|
|
@@ -1,88 +0,0 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
// F34 hidden verifier — file-order settlement under inverted completion order.
|
|
3
|
-
// Transfer ids encode gateway latency (trailing digits = ms), so the FIRST
|
|
4
|
-
// transfer in the file finishes authorization LAST. Applying balances in
|
|
5
|
-
// completion order (or rejecting against pre-batch balances) flips t-60 and
|
|
6
|
-
// t-40; only strict file-order settlement produces this exact output.
|
|
7
|
-
|
|
8
|
-
const fs = require('node:fs');
|
|
9
|
-
const os = require('node:os');
|
|
10
|
-
const path = require('node:path');
|
|
11
|
-
const { spawnSync } = require('node:child_process');
|
|
12
|
-
|
|
13
|
-
// Structural equality: object key order is NOT part of the visible contract;
|
|
14
|
-
// array order IS. Codex fairness-audit fix (stringify equality made JSON
|
|
15
|
-
// property order a hidden oracle narrower than spec.md).
|
|
16
|
-
function deepEqual(a, b) {
|
|
17
|
-
if (a === b) return true;
|
|
18
|
-
if (Array.isArray(a) && Array.isArray(b)) {
|
|
19
|
-
return a.length === b.length && a.every((v, i) => deepEqual(v, b[i]));
|
|
20
|
-
}
|
|
21
|
-
if (a && b && typeof a === 'object' && typeof b === 'object' && !Array.isArray(a) && !Array.isArray(b)) {
|
|
22
|
-
const ka = Object.keys(a).sort();
|
|
23
|
-
const kb = Object.keys(b).sort();
|
|
24
|
-
return deepEqual(ka, kb) && ka.every((k) => deepEqual(a[k], b[k]));
|
|
25
|
-
}
|
|
26
|
-
return false;
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
const inputPath = path.join(os.tmpdir(), `settle-input-${process.pid}.json`);
|
|
30
|
-
fs.writeFileSync(inputPath, JSON.stringify({
|
|
31
|
-
transfers: [
|
|
32
|
-
{ id: 't-80', from: 'acct-a', to: 'acct-b', amount: 70 },
|
|
33
|
-
{ id: 't-60', from: 'acct-b', to: 'acct-c', amount: 50 },
|
|
34
|
-
{ id: 't-40', from: 'acct-a', to: 'acct-d', amount: 40 },
|
|
35
|
-
{ id: 't-20', from: 'acct-c', to: 'acct-a', amount: 100 },
|
|
36
|
-
{ id: 't-10', from: 'acct-d', to: 'acct-b', amount: 5 },
|
|
37
|
-
{ id: 't-5', from: 'acct-b', to: 'acct-a', amount: 25 }
|
|
38
|
-
]
|
|
39
|
-
}));
|
|
40
|
-
|
|
41
|
-
const work = process.env.BENCH_WORKDIR;
|
|
42
|
-
const cli = path.join(work, 'bin', 'cli.js');
|
|
43
|
-
const result = spawnSync('node', [cli, 'settle', '--input', inputPath], {
|
|
44
|
-
cwd: work,
|
|
45
|
-
encoding: 'utf8'
|
|
46
|
-
});
|
|
47
|
-
|
|
48
|
-
let out;
|
|
49
|
-
try {
|
|
50
|
-
out = JSON.parse(result.stdout);
|
|
51
|
-
} catch {
|
|
52
|
-
out = null;
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
const expected = {
|
|
56
|
-
accepted: ['t-80', 't-60', 't-20', 't-10', 't-5'],
|
|
57
|
-
rejected: [
|
|
58
|
-
{ id: 't-40', error: 'insufficient_funds', available: 30, requested: 40 }
|
|
59
|
-
],
|
|
60
|
-
balances: { 'acct-a': 155, 'acct-b': 0, 'acct-c': 0, 'acct-d': 5 }
|
|
61
|
-
};
|
|
62
|
-
|
|
63
|
-
let stats;
|
|
64
|
-
try {
|
|
65
|
-
stats = JSON.parse(fs.readFileSync(path.join(work, 'data', 'gateway-stats.json'), 'utf8'));
|
|
66
|
-
} catch {
|
|
67
|
-
stats = null;
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
const accountsUntouched = (() => {
|
|
71
|
-
try {
|
|
72
|
-
const seeded = JSON.parse(fs.readFileSync(path.join(work, 'data', 'accounts.json'), 'utf8'));
|
|
73
|
-
return deepEqual(seeded, { 'acct-a': 100, 'acct-b': 0, 'acct-c': 50, 'acct-d': 10 });
|
|
74
|
-
} catch {
|
|
75
|
-
return false;
|
|
76
|
-
}
|
|
77
|
-
})();
|
|
78
|
-
|
|
79
|
-
const ok = result.status === 0
|
|
80
|
-
&& result.stderr === ''
|
|
81
|
-
&& deepEqual(out, expected)
|
|
82
|
-
&& stats !== null
|
|
83
|
-
&& stats.calls === 6
|
|
84
|
-
&& stats.high_water >= 3
|
|
85
|
-
&& accountsUntouched;
|
|
86
|
-
|
|
87
|
-
console.log(JSON.stringify({ ok, status: result.status, stderr: result.stderr, out, stats, accountsUntouched }));
|
|
88
|
-
process.exit(ok ? 0 : 1);
|
package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/verifiers/reject-invalid.js
DELETED
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
// F34 hidden verifier — validation happens before any authorization: an
|
|
3
|
-
// unknown account exits 2 with one JSON error object on stderr, no stdout,
|
|
4
|
-
// and the gateway records zero calls for the run.
|
|
5
|
-
|
|
6
|
-
const fs = require('node:fs');
|
|
7
|
-
const os = require('node:os');
|
|
8
|
-
const path = require('node:path');
|
|
9
|
-
const { spawnSync } = require('node:child_process');
|
|
10
|
-
|
|
11
|
-
const work = process.env.BENCH_WORKDIR;
|
|
12
|
-
const statsPath = path.join(work, 'data', 'gateway-stats.json');
|
|
13
|
-
fs.rmSync(statsPath, { force: true });
|
|
14
|
-
|
|
15
|
-
const inputPath = path.join(os.tmpdir(), `settle-bad-${process.pid}.json`);
|
|
16
|
-
fs.writeFileSync(inputPath, JSON.stringify({
|
|
17
|
-
transfers: [
|
|
18
|
-
{ id: 't-30', from: 'acct-a', to: 'acct-zzz', amount: 10 }
|
|
19
|
-
]
|
|
20
|
-
}));
|
|
21
|
-
|
|
22
|
-
const cli = path.join(work, 'bin', 'cli.js');
|
|
23
|
-
const result = spawnSync('node', [cli, 'settle', '--input', inputPath], {
|
|
24
|
-
cwd: work,
|
|
25
|
-
encoding: 'utf8'
|
|
26
|
-
});
|
|
27
|
-
|
|
28
|
-
let err;
|
|
29
|
-
try {
|
|
30
|
-
err = JSON.parse(result.stderr);
|
|
31
|
-
} catch {
|
|
32
|
-
err = null;
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
const gatewayNeverCalled = !fs.existsSync(statsPath);
|
|
36
|
-
|
|
37
|
-
const ok = result.status === 2
|
|
38
|
-
&& result.stdout === ''
|
|
39
|
-
&& err !== null
|
|
40
|
-
&& err.error === 'invalid_input'
|
|
41
|
-
&& typeof err.reason === 'string'
|
|
42
|
-
&& gatewayNeverCalled;
|
|
43
|
-
|
|
44
|
-
console.log(JSON.stringify({ ok, status: result.status, stdout: result.stdout, err, gatewayNeverCalled }));
|
|
45
|
-
process.exit(ok ? 0 : 1);
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
# F35 CLI apply journal
|
|
2
|
-
|
|
3
|
-
## Failure mode
|
|
4
|
-
|
|
5
|
-
Detects implementations that flatten the composite ordering/replay/atomicity
|
|
6
|
-
contract into the easy shape: apply-in-file-order against pre-batch stock
|
|
7
|
-
with per-row persistence. Three discriminators: (a) `applied` order and
|
|
8
|
-
journal append order must follow priority-desc with file-order ties (final
|
|
9
|
-
quantities alone cannot catch this — addition commutes); (b) the failing
|
|
10
|
-
op's `available` is measured at its turn AFTER higher-priority ops changed
|
|
11
|
-
stock (rollback verifier: 15, not the pre-batch 10); (c) any failure leaves
|
|
12
|
-
both files byte-for-byte unchanged. In-file duplicate `op_id` is a
|
|
13
|
-
validation error while a journaled `op_id` is a per-op skip — the
|
|
14
|
-
distinction is the replay axis (Tier-1 "A1" from memory
|
|
15
|
-
`project_pair_fixture_tier1_2026_05_05`; keyword-hidden: spec never says
|
|
16
|
-
atomic/transaction/idempotent/rollback).
|
|
17
|
-
|
|
18
|
-
## Fairness
|
|
19
|
-
|
|
20
|
-
Every verifier assertion quotes a visible spec bullet via `contract_refs`
|
|
21
|
-
(lint-fixtures enforced). Verifiers reset the seeded files before running so
|
|
22
|
-
command order cannot leak state between checks. No hidden shape, tie-break,
|
|
23
|
-
or availability rule exists outside spec.md.
|
|
24
|
-
|
|
25
|
-
## Headroom prediction (pre-registered, unmeasured)
|
|
26
|
-
|
|
27
|
-
bare ≤ 60 (expected to miss at-its-turn availability or journal append
|
|
28
|
-
order), solo ≤ 80 (hypothesis: evaluates availability against pre-batch or
|
|
29
|
-
file-order state). To be measured with `headroom-gate.py` before any
|
|
30
|
-
pair-evidence claims; this fixture carries no measurement claim until then.
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
# F35 retired
|
|
2
|
-
|
|
3
|
-
Retired from the active golden suite after headroom run
|
|
4
|
-
`iter-0039-headroom-f34-f35`.
|
|
5
|
-
|
|
6
|
-
Reason: `solo_claude` scored 97, exceeding the headroom ceiling of 80, while
|
|
7
|
-
bare scored 50. `solo_claude` correctly implemented the `{ "ops": [...] }`
|
|
8
|
-
input contract, priority ordering, rollback, and duplicate-op handling; bare
|
|
9
|
-
failed only because it assumed a bare top-level array instead of the required
|
|
10
|
-
object shape — an input-shape slip, not evidence the task is hard. The
|
|
11
|
-
solo_claude arm also ran to the full 1500s timeout before finishing
|
|
12
|
-
(elapsed_seconds 1501, INVOKE_EXIT=124), but its diff still verified clean, so
|
|
13
|
-
the fixture is not pair-lift evidence.
|
|
14
|
-
|
|
15
|
-
Future use: rework the visible contract or hidden verifiers so the task
|
|
16
|
-
creates a fair pair-risk-probe gap without hiding requirements from the spec.
|
|
@@ -1,64 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"verification_commands": [
|
|
3
|
-
{
|
|
4
|
-
"cmd": "node --test tests/cli.test.js",
|
|
5
|
-
"exit_code": 0,
|
|
6
|
-
"stdout_contains": [],
|
|
7
|
-
"stdout_not_contains": ["not ok "]
|
|
8
|
-
},
|
|
9
|
-
{
|
|
10
|
-
"cmd": "node \"$BENCH_FIXTURE_DIR/verifiers/priority-apply.js\"",
|
|
11
|
-
"exit_code": 0,
|
|
12
|
-
"stdout_contains": ["\"ok\":true"],
|
|
13
|
-
"stdout_not_contains": [],
|
|
14
|
-
"contract_refs": [
|
|
15
|
-
"An op whose `op_id` is already listed in the journal's `applied` array is recorded as skipped and changes nothing, regardless of its other fields.",
|
|
16
|
-
"Non-skipped ops apply in descending `priority`; ties apply in file order.",
|
|
17
|
-
"On success, persist the final quantities to `data/inventory.json`, append the applied `op_id`s to the journal's `applied` array in application order, and write exactly one JSON object to stdout with keys `applied` (op_ids in application order), `skipped` (op_ids in file order), and `inventory` (final quantities for every sku)."
|
|
18
|
-
]
|
|
19
|
-
},
|
|
20
|
-
{
|
|
21
|
-
"cmd": "node \"$BENCH_FIXTURE_DIR/verifiers/rollback.js\"",
|
|
22
|
-
"exit_code": 0,
|
|
23
|
-
"stdout_contains": ["\"ok\":true"],
|
|
24
|
-
"stdout_not_contains": [],
|
|
25
|
-
"contract_refs": [
|
|
26
|
-
"An op fails when applying its `delta` would take the sku's quantity below zero, evaluated at its turn in that order, with exact error shape `{ \"error\": \"insufficient_stock\", \"op_id\": string, \"sku\": string, \"available\": number, \"requested\": number }` where `available` is the quantity at that op's turn and `requested` is the absolute value of its `delta`.",
|
|
27
|
-
"If any op fails, nothing persists: `data/inventory.json` and `data/journal.json` are byte-for-byte unchanged, the command exits `2`, and exactly one JSON error object is written to stderr with nothing on stdout."
|
|
28
|
-
]
|
|
29
|
-
},
|
|
30
|
-
{
|
|
31
|
-
"cmd": "node \"$BENCH_FIXTURE_DIR/verifiers/dup-op.js\"",
|
|
32
|
-
"exit_code": 0,
|
|
33
|
-
"stdout_contains": ["\"ok\":true"],
|
|
34
|
-
"stdout_not_contains": [],
|
|
35
|
-
"contract_refs": [
|
|
36
|
-
"or an `op_id` that appears more than once in the input file exits `2` and writes exactly one JSON error object to stderr with shape `{ \"error\": \"invalid_input\", \"reason\": string }`, printing nothing to stdout and changing neither file."
|
|
37
|
-
]
|
|
38
|
-
}
|
|
39
|
-
],
|
|
40
|
-
"forbidden_patterns": [
|
|
41
|
-
{
|
|
42
|
-
"pattern": "catch\\s*\\([^)]*\\)\\s*\\{[^}]*return\\s+(null|undefined|''|\\{\\})",
|
|
43
|
-
"description": "silent catch returning fallback in apply path",
|
|
44
|
-
"files": ["bin/cli.js"],
|
|
45
|
-
"severity": "disqualifier"
|
|
46
|
-
},
|
|
47
|
-
{
|
|
48
|
-
"pattern": "catch\\s*\\([^)]*\\)\\s*\\{\\s*\\}",
|
|
49
|
-
"description": "empty catch block",
|
|
50
|
-
"files": ["bin/cli.js"],
|
|
51
|
-
"severity": "disqualifier"
|
|
52
|
-
},
|
|
53
|
-
{
|
|
54
|
-
"pattern": "/\\*\\s*eslint-disable",
|
|
55
|
-
"description": "eslint-disable without scoped justification",
|
|
56
|
-
"files": ["bin/cli.js"],
|
|
57
|
-
"severity": "disqualifier"
|
|
58
|
-
}
|
|
59
|
-
],
|
|
60
|
-
"required_files": ["bin/cli.js", "tests/cli.test.js", "data/inventory.json", "data/journal.json"],
|
|
61
|
-
"forbidden_files": [],
|
|
62
|
-
"max_deps_added": 0,
|
|
63
|
-
"spec_output_files": ["bin/cli.js", "tests/cli.test.js"]
|
|
64
|
-
}
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "F35-cli-apply-journal",
|
|
3
|
-
"category": "high-risk",
|
|
4
|
-
"difficulty": "high",
|
|
5
|
-
"timeout_seconds": 1500,
|
|
6
|
-
"required_tools": ["node"],
|
|
7
|
-
"browser": false,
|
|
8
|
-
"deps_change_expected": false,
|
|
9
|
-
"intent": "Add a bench-cli apply command that applies inventory adjustments in priority order with journal-based replay skipping, at-its-turn stock checks, exact JSON output, and all-or-nothing persistence of inventory and journal."
|
|
10
|
-
}
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# F35 setup — seed inventory and the replay journal.
|
|
3
|
-
set -e
|
|
4
|
-
|
|
5
|
-
mkdir -p data
|
|
6
|
-
|
|
7
|
-
cat > data/inventory.json <<'JSON'
|
|
8
|
-
{
|
|
9
|
-
"widget": 10,
|
|
10
|
-
"gadget": 4,
|
|
11
|
-
"gizmo": 0
|
|
12
|
-
}
|
|
13
|
-
JSON
|
|
14
|
-
|
|
15
|
-
cat > data/journal.json <<'JSON'
|
|
16
|
-
{
|
|
17
|
-
"applied": ["op-900"]
|
|
18
|
-
}
|
|
19
|
-
JSON
|
|
20
|
-
|
|
21
|
-
exit 0
|
|
@@ -1,58 +0,0 @@
|
|
|
1
|
-
---
|
|
2
|
-
id: "F35-cli-apply-journal"
|
|
3
|
-
title: "Apply command with journal replay and all-or-nothing persistence"
|
|
4
|
-
status: planned
|
|
5
|
-
complexity: high
|
|
6
|
-
depends-on: []
|
|
7
|
-
---
|
|
8
|
-
|
|
9
|
-
# F35 Apply command with journal replay and all-or-nothing persistence
|
|
10
|
-
|
|
11
|
-
## Context
|
|
12
|
-
|
|
13
|
-
`bench-cli` gets an `apply` command that applies a file of inventory
|
|
14
|
-
adjustments to `data/inventory.json`. Runs are re-deliverable: a journal at
|
|
15
|
-
`data/journal.json` records which ops have already been applied, so a
|
|
16
|
-
delivered-again op must not change inventory twice. A batch either fully
|
|
17
|
-
persists or leaves both files untouched.
|
|
18
|
-
|
|
19
|
-
## Requirements
|
|
20
|
-
|
|
21
|
-
- [ ] `bench-cli apply --input <path>` reads JSON shaped as `{ "ops": [{ "op_id": string, "sku": string, "delta": number, "priority": number }] }`.
|
|
22
|
-
- [ ] Quantities come from `data/inventory.json`; the applied-op record comes from the journal's `applied` array in `data/journal.json`. Both are seeded by setup.
|
|
23
|
-
- [ ] Validation happens before any application. Invalid JSON, missing or non-string `op_id`/`sku`, unknown `sku`, non-integer `delta` or `priority`, or an `op_id` that appears more than once in the input file exits `2` and writes exactly one JSON error object to stderr with shape `{ "error": "invalid_input", "reason": string }`, printing nothing to stdout and changing neither file.
|
|
24
|
-
- [ ] An op whose `op_id` is already listed in the journal's `applied` array is recorded as skipped and changes nothing, regardless of its other fields.
|
|
25
|
-
- [ ] Non-skipped ops apply in descending `priority`; ties apply in file order.
|
|
26
|
-
- [ ] An op fails when applying its `delta` would take the sku's quantity below zero, evaluated at its turn in that order, with exact error shape `{ "error": "insufficient_stock", "op_id": string, "sku": string, "available": number, "requested": number }` where `available` is the quantity at that op's turn and `requested` is the absolute value of its `delta`.
|
|
27
|
-
- [ ] If any op fails, nothing persists: `data/inventory.json` and `data/journal.json` are byte-for-byte unchanged, the command exits `2`, and exactly one JSON error object is written to stderr with nothing on stdout.
|
|
28
|
-
- [ ] On success, persist the final quantities to `data/inventory.json`, append the applied `op_id`s to the journal's `applied` array in application order, and write exactly one JSON object to stdout with keys `applied` (op_ids in application order), `skipped` (op_ids in file order), and `inventory` (final quantities for every sku). Nothing is written to stderr on success.
|
|
29
|
-
- [ ] `tests/cli.test.js` is updated. Existing tests still pass AND at least two new tests cover `apply`: one success mixing a journaled op with different priorities, and one failing batch that leaves both files unchanged.
|
|
30
|
-
|
|
31
|
-
## Constraints
|
|
32
|
-
|
|
33
|
-
- **No new npm dependencies.**
|
|
34
|
-
- **No silent catches.** If parsing or file reading fails, emit a visible JSON error to stderr and exit `2`.
|
|
35
|
-
- **No extra stdout/stderr text** on the success path; downstream tooling parses stdout as JSON.
|
|
36
|
-
- Quantities are integers; no fractional inventory.
|
|
37
|
-
|
|
38
|
-
## Out of Scope
|
|
39
|
-
|
|
40
|
-
- Changing server routes, web UI, or journal schema.
|
|
41
|
-
- Concurrency across multiple `apply` processes.
|
|
42
|
-
- Pruning or rewriting existing journal entries.
|
|
43
|
-
- Touching `server/`, `web/`, or `tests/server.test.js`.
|
|
44
|
-
|
|
45
|
-
## Verification
|
|
46
|
-
|
|
47
|
-
- `node --test tests/cli.test.js` exits 0.
|
|
48
|
-
- A batch mixing a journaled op, unequal priorities, and a priority tie yields exact `applied` order, exact `skipped`, exact final `inventory`, and the journal grows by exactly the applied op_ids in that order.
|
|
49
|
-
- A batch whose failing op is only discoverable after higher-priority ops have consumed or added stock reports `available` at that op's turn and leaves both files byte-for-byte unchanged.
|
|
50
|
-
- A duplicated `op_id` in the input file exits `2` with one JSON error on stderr and no stdout.
|
|
51
|
-
- `git diff --stat` shows only `bin/cli.js` and `tests/cli.test.js` touched (inventory and journal are seeded by setup, not the arm).
|
|
52
|
-
|
|
53
|
-
## Solo-headroom hypothesis
|
|
54
|
-
|
|
55
|
-
A capable solo_claude baseline is expected to miss that each op's
|
|
56
|
-
`available` stock is evaluated at its turn in priority order rather than
|
|
57
|
-
against the pre-batch inventory or file order; observable command
|
|
58
|
-
`node "$BENCH_FIXTURE_DIR/verifiers/rollback.js"` exposes the miss.
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
Add an `apply` command to `bench-cli` so users can run `bench-cli apply --input <path>` with an ops JSON file of inventory adjustments. Quantities live in `data/inventory.json` and previously applied op ids live in the `applied` array of `data/journal.json` (both seeded).
|
|
2
|
-
|
|
3
|
-
Validate the whole file before applying anything: bad JSON, missing/invalid fields, unknown sku, non-integer delta or priority, or the same `op_id` appearing twice in one file exits `2` with exactly one JSON error object `{ "error": "invalid_input", "reason": string }` on stderr, nothing on stdout, and neither file changed.
|
|
4
|
-
|
|
5
|
-
An op whose `op_id` is already in the journal is recorded as skipped and changes nothing — whatever its other fields say. Everything else applies in descending `priority` (ties in file order). An op fails if its `delta` would push the sku below zero at its turn in that order; the error is the exact object `{ "error": "insufficient_stock", "op_id", "sku", "available", "requested" }` with `available` measured at that op's turn. Any failure means nothing persists — inventory and journal stay byte-for-byte identical and the command exits `2`.
|
|
6
|
-
|
|
7
|
-
On success, write the final quantities back to `data/inventory.json`, append applied op ids to the journal in application order, and print exactly one JSON object with `applied` (application order), `skipped` (file order), and `inventory` — with nothing on stderr.
|
|
8
|
-
|
|
9
|
-
Update `tests/cli.test.js`: existing tests still pass, plus at least two apply tests (one success mixing a journaled op with different priorities, one failing batch that leaves both files unchanged). No new npm dependencies. Only touch `bin/cli.js` and `tests/cli.test.js`.
|
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
// F35 hidden verifier — an op_id duplicated WITHIN the input file is a
|
|
3
|
-
// validation error (whole file rejected, nothing changes), distinct from a
|
|
4
|
-
// journal replay which is a per-op skip. Exit 2, one JSON error object on
|
|
5
|
-
// stderr, no stdout, both files untouched.
|
|
6
|
-
|
|
7
|
-
const fs = require('node:fs');
|
|
8
|
-
const os = require('node:os');
|
|
9
|
-
const path = require('node:path');
|
|
10
|
-
const { spawnSync } = require('node:child_process');
|
|
11
|
-
|
|
12
|
-
const work = process.env.BENCH_WORKDIR;
|
|
13
|
-
const invPath = path.join(work, 'data', 'inventory.json');
|
|
14
|
-
const jrnPath = path.join(work, 'data', 'journal.json');
|
|
15
|
-
|
|
16
|
-
fs.writeFileSync(invPath, JSON.stringify({ widget: 10, gadget: 4, gizmo: 0 }, null, 2) + '\n');
|
|
17
|
-
fs.writeFileSync(jrnPath, JSON.stringify({ applied: ['op-900'] }, null, 2) + '\n');
|
|
18
|
-
const invBefore = fs.readFileSync(invPath, 'utf8');
|
|
19
|
-
const jrnBefore = fs.readFileSync(jrnPath, 'utf8');
|
|
20
|
-
|
|
21
|
-
const inputPath = path.join(os.tmpdir(), `apply-dup-${process.pid}.json`);
|
|
22
|
-
fs.writeFileSync(inputPath, JSON.stringify({
|
|
23
|
-
ops: [
|
|
24
|
-
{ op_id: 'op-20', sku: 'widget', delta: -1, priority: 3 },
|
|
25
|
-
{ op_id: 'op-21', sku: 'gadget', delta: 1, priority: 2 },
|
|
26
|
-
{ op_id: 'op-20', sku: 'gizmo', delta: 2, priority: 1 }
|
|
27
|
-
]
|
|
28
|
-
}));
|
|
29
|
-
|
|
30
|
-
const cli = path.join(work, 'bin', 'cli.js');
|
|
31
|
-
const result = spawnSync('node', [cli, 'apply', '--input', inputPath], {
|
|
32
|
-
cwd: work,
|
|
33
|
-
encoding: 'utf8'
|
|
34
|
-
});
|
|
35
|
-
|
|
36
|
-
let err;
|
|
37
|
-
try {
|
|
38
|
-
err = JSON.parse(result.stderr);
|
|
39
|
-
} catch {
|
|
40
|
-
err = null;
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
const filesUnchanged = fs.readFileSync(invPath, 'utf8') === invBefore
|
|
44
|
-
&& fs.readFileSync(jrnPath, 'utf8') === jrnBefore;
|
|
45
|
-
|
|
46
|
-
const ok = result.status === 2
|
|
47
|
-
&& result.stdout === ''
|
|
48
|
-
&& err !== null
|
|
49
|
-
&& err.error === 'invalid_input'
|
|
50
|
-
&& typeof err.reason === 'string'
|
|
51
|
-
&& filesUnchanged;
|
|
52
|
-
|
|
53
|
-
console.log(JSON.stringify({ ok, status: result.status, stdout: result.stdout, err, filesUnchanged }));
|
|
54
|
-
process.exit(ok ? 0 : 1);
|
package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/verifiers/priority-apply.js
DELETED
|
@@ -1,82 +0,0 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
// F35 hidden verifier — success path mixing a journaled (replayed) op, unequal
|
|
3
|
-
// priorities, and a priority tie. Final quantities are order-independent
|
|
4
|
-
// (addition commutes), so the discriminator is the exact `applied` ORDER and
|
|
5
|
-
// the journal append order: priority-desc with file-order ties, replayed op
|
|
6
|
-
// skipped regardless of its (otherwise failing) fields.
|
|
7
|
-
|
|
8
|
-
const fs = require('node:fs');
|
|
9
|
-
const os = require('node:os');
|
|
10
|
-
const path = require('node:path');
|
|
11
|
-
const { spawnSync } = require('node:child_process');
|
|
12
|
-
|
|
13
|
-
// Structural equality: object key order is NOT part of the visible contract;
|
|
14
|
-
// array order IS. Codex fairness-audit fix (stringify equality made JSON
|
|
15
|
-
// property order a hidden oracle narrower than spec.md).
|
|
16
|
-
function deepEqual(a, b) {
|
|
17
|
-
if (a === b) return true;
|
|
18
|
-
if (Array.isArray(a) && Array.isArray(b)) {
|
|
19
|
-
return a.length === b.length && a.every((v, i) => deepEqual(v, b[i]));
|
|
20
|
-
}
|
|
21
|
-
if (a && b && typeof a === 'object' && typeof b === 'object' && !Array.isArray(a) && !Array.isArray(b)) {
|
|
22
|
-
const ka = Object.keys(a).sort();
|
|
23
|
-
const kb = Object.keys(b).sort();
|
|
24
|
-
return deepEqual(ka, kb) && ka.every((k) => deepEqual(a[k], b[k]));
|
|
25
|
-
}
|
|
26
|
-
return false;
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
const work = process.env.BENCH_WORKDIR;
|
|
30
|
-
const invPath = path.join(work, 'data', 'inventory.json');
|
|
31
|
-
const jrnPath = path.join(work, 'data', 'journal.json');
|
|
32
|
-
|
|
33
|
-
// Reset seeds so this verifier is order-independent from other commands.
|
|
34
|
-
fs.writeFileSync(invPath, JSON.stringify({ widget: 10, gadget: 4, gizmo: 0 }, null, 2) + '\n');
|
|
35
|
-
fs.writeFileSync(jrnPath, JSON.stringify({ applied: ['op-900'] }, null, 2) + '\n');
|
|
36
|
-
|
|
37
|
-
const inputPath = path.join(os.tmpdir(), `apply-ok-${process.pid}.json`);
|
|
38
|
-
fs.writeFileSync(inputPath, JSON.stringify({
|
|
39
|
-
ops: [
|
|
40
|
-
{ op_id: 'op-1', sku: 'widget', delta: -4, priority: 1 },
|
|
41
|
-
{ op_id: 'op-900', sku: 'widget', delta: -9, priority: 9 },
|
|
42
|
-
{ op_id: 'op-2', sku: 'widget', delta: -5, priority: 5 },
|
|
43
|
-
{ op_id: 'op-3', sku: 'gadget', delta: -4, priority: 5 },
|
|
44
|
-
{ op_id: 'op-4', sku: 'widget', delta: 3, priority: 0 }
|
|
45
|
-
]
|
|
46
|
-
}));
|
|
47
|
-
|
|
48
|
-
const cli = path.join(work, 'bin', 'cli.js');
|
|
49
|
-
const result = spawnSync('node', [cli, 'apply', '--input', inputPath], {
|
|
50
|
-
cwd: work,
|
|
51
|
-
encoding: 'utf8'
|
|
52
|
-
});
|
|
53
|
-
|
|
54
|
-
let out;
|
|
55
|
-
try {
|
|
56
|
-
out = JSON.parse(result.stdout);
|
|
57
|
-
} catch {
|
|
58
|
-
out = null;
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
const expected = {
|
|
62
|
-
applied: ['op-2', 'op-3', 'op-1', 'op-4'],
|
|
63
|
-
skipped: ['op-900'],
|
|
64
|
-
inventory: { widget: 4, gadget: 0, gizmo: 0 }
|
|
65
|
-
};
|
|
66
|
-
|
|
67
|
-
let inv; let jrn;
|
|
68
|
-
try {
|
|
69
|
-
inv = JSON.parse(fs.readFileSync(invPath, 'utf8'));
|
|
70
|
-
jrn = JSON.parse(fs.readFileSync(jrnPath, 'utf8'));
|
|
71
|
-
} catch {
|
|
72
|
-
inv = null; jrn = null;
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
const ok = result.status === 0
|
|
76
|
-
&& result.stderr === ''
|
|
77
|
-
&& deepEqual(out, expected)
|
|
78
|
-
&& deepEqual(inv, expected.inventory)
|
|
79
|
-
&& deepEqual(jrn, { applied: ['op-900', 'op-2', 'op-3', 'op-1', 'op-4'] });
|
|
80
|
-
|
|
81
|
-
console.log(JSON.stringify({ ok, status: result.status, stderr: result.stderr, out, inv, jrn }));
|
|
82
|
-
process.exit(ok ? 0 : 1);
|
|
@@ -1,77 +0,0 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
// F35 hidden verifier — the failing op is only discoverable AFTER a
|
|
3
|
-
// higher-priority op has added stock: op-12 (priority 8) raises widget
|
|
4
|
-
// 10 -> 15, then op-11 (priority 5) requests 20 and fails with
|
|
5
|
-
// available 15 — not 10 (pre-batch / file-order would report 10).
|
|
6
|
-
// Both files must be byte-for-byte unchanged afterward.
|
|
7
|
-
|
|
8
|
-
const fs = require('node:fs');
|
|
9
|
-
const os = require('node:os');
|
|
10
|
-
const path = require('node:path');
|
|
11
|
-
const { spawnSync } = require('node:child_process');
|
|
12
|
-
|
|
13
|
-
// Structural equality: object key order is NOT part of the visible contract;
|
|
14
|
-
// array order IS. Codex fairness-audit fix (stringify equality made JSON
|
|
15
|
-
// property order a hidden oracle narrower than spec.md).
|
|
16
|
-
function deepEqual(a, b) {
|
|
17
|
-
if (a === b) return true;
|
|
18
|
-
if (Array.isArray(a) && Array.isArray(b)) {
|
|
19
|
-
return a.length === b.length && a.every((v, i) => deepEqual(v, b[i]));
|
|
20
|
-
}
|
|
21
|
-
if (a && b && typeof a === 'object' && typeof b === 'object' && !Array.isArray(a) && !Array.isArray(b)) {
|
|
22
|
-
const ka = Object.keys(a).sort();
|
|
23
|
-
const kb = Object.keys(b).sort();
|
|
24
|
-
return deepEqual(ka, kb) && ka.every((k) => deepEqual(a[k], b[k]));
|
|
25
|
-
}
|
|
26
|
-
return false;
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
const work = process.env.BENCH_WORKDIR;
|
|
30
|
-
const invPath = path.join(work, 'data', 'inventory.json');
|
|
31
|
-
const jrnPath = path.join(work, 'data', 'journal.json');
|
|
32
|
-
|
|
33
|
-
fs.writeFileSync(invPath, JSON.stringify({ widget: 10, gadget: 4, gizmo: 0 }, null, 2) + '\n');
|
|
34
|
-
fs.writeFileSync(jrnPath, JSON.stringify({ applied: ['op-900'] }, null, 2) + '\n');
|
|
35
|
-
const invBefore = fs.readFileSync(invPath, 'utf8');
|
|
36
|
-
const jrnBefore = fs.readFileSync(jrnPath, 'utf8');
|
|
37
|
-
|
|
38
|
-
const inputPath = path.join(os.tmpdir(), `apply-fail-${process.pid}.json`);
|
|
39
|
-
fs.writeFileSync(inputPath, JSON.stringify({
|
|
40
|
-
ops: [
|
|
41
|
-
{ op_id: 'op-10', sku: 'gadget', delta: 2, priority: 9 },
|
|
42
|
-
{ op_id: 'op-11', sku: 'widget', delta: -20, priority: 5 },
|
|
43
|
-
{ op_id: 'op-12', sku: 'widget', delta: 5, priority: 8 }
|
|
44
|
-
]
|
|
45
|
-
}));
|
|
46
|
-
|
|
47
|
-
const cli = path.join(work, 'bin', 'cli.js');
|
|
48
|
-
const result = spawnSync('node', [cli, 'apply', '--input', inputPath], {
|
|
49
|
-
cwd: work,
|
|
50
|
-
encoding: 'utf8'
|
|
51
|
-
});
|
|
52
|
-
|
|
53
|
-
let err;
|
|
54
|
-
try {
|
|
55
|
-
err = JSON.parse(result.stderr);
|
|
56
|
-
} catch {
|
|
57
|
-
err = null;
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
const expectedErr = {
|
|
61
|
-
error: 'insufficient_stock',
|
|
62
|
-
op_id: 'op-11',
|
|
63
|
-
sku: 'widget',
|
|
64
|
-
available: 15,
|
|
65
|
-
requested: 20
|
|
66
|
-
};
|
|
67
|
-
|
|
68
|
-
const filesUnchanged = fs.readFileSync(invPath, 'utf8') === invBefore
|
|
69
|
-
&& fs.readFileSync(jrnPath, 'utf8') === jrnBefore;
|
|
70
|
-
|
|
71
|
-
const ok = result.status === 2
|
|
72
|
-
&& result.stdout === ''
|
|
73
|
-
&& deepEqual(err, expectedErr)
|
|
74
|
-
&& filesUnchanged;
|
|
75
|
-
|
|
76
|
-
console.log(JSON.stringify({ ok, status: result.status, stdout: result.stdout, err, filesUnchanged }));
|
|
77
|
-
process.exit(ok ? 0 : 1);
|