devlyn-cli 2.3.0 → 2.3.2

package/benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh

@@ -28,7 +28,14 @@ EOF
   cat > "$TMP_DIR/$run_id/compare.json" <<EOF
 {
   "solo": {"invoke_exit": 0, "timed_out": false, "verify_verdict": "$solo_verdict", "elapsed_seconds": 100},
-  "pair": {"invoke_exit": 0, "timed_out": false, "verify_verdict": "$pair_verdict", "pair_mode": true, "elapsed_seconds": 200},
+  "pair": {
+    "invoke_exit": 0,
+    "timed_out": false,
+    "verify_verdict": "$pair_verdict",
+    "pair_mode": true,
+    "pair_trigger": {"eligible": true, "reasons": ["mode.verify-only"], "skipped_reason": null},
+    "elapsed_seconds": 200
+  },
   "comparison": {
     "pair_trigger_missed": false,
     "pair_verdict_lift": $lift,
@@ -64,10 +71,24 @@ write_run pass-b F12-webhook-raw-body-signature PASS_WITH_ISSUES NEEDS_WORK true
 mkdir -p "$FIXTURES_DIR/F10-persist-write-collision" "$FIXTURES_DIR/F12-webhook-raw-body-signature"
 python3 "$GATE" --results-root "$TMP_DIR" --fixtures-root "$FIXTURES_DIR" \
   --run-id pass-a --run-id pass-b --min-runs 2 --max-pair-solo-wall-ratio 3 \
+  --out-md "$TMP_DIR/pass.md" \
   > "$TMP_DIR/pass.out"
 grep -Fq '"verdict": "PASS"' "$TMP_DIR/pass.out"
 grep -Fq '"avg_pair_solo_wall_ratio": 2.0' "$TMP_DIR/pass.out"
 grep -Fq '"pair_solo_wall_ratio": 2.0' "$TMP_DIR/pass.out"
+grep -Fq '"pair_trigger_reasons": [' "$TMP_DIR/pass.out"
+grep -Fq '"mode.verify-only"' "$TMP_DIR/pass.out"
+grep -Fq '"pair_trigger_has_canonical_reason": true' "$TMP_DIR/pass.out"
+grep -Fq '| Run | Fixture | Solo VERIFY | Pair VERIFY | Pair mode | Triggers | Wall ratio | External lift | Internal lift | Status | Reason |' "$TMP_DIR/pass.md"
+grep -Fq '| pass-a | F10-persist-write-collision | PASS_WITH_ISSUES | NEEDS_WORK | true | mode.verify-only | 2.00x | true | false | PASS | ok |' "$TMP_DIR/pass.md"
+
+expect_fail_contains invalid-min-runs "value must be > 0" \
+  python3 "$GATE" --results-root "$TMP_DIR" --fixtures-root "$FIXTURES_DIR" \
+    --run-id pass-a --min-runs 0
+
+expect_fail_contains invalid-wall-ratio "value must be > 0" \
+  python3 "$GATE" --results-root "$TMP_DIR" --fixtures-root "$FIXTURES_DIR" \
+    --run-id pass-a --max-pair-solo-wall-ratio 0
 
 mkdir -p "$TMP_DIR/summary-verdicts/pair"
 cat > "$TMP_DIR/summary-verdicts/pair/input.md" <<'EOF'
@@ -76,7 +97,14 @@ EOF
 cat > "$TMP_DIR/summary-verdicts/compare.json" <<'EOF'
 {
   "solo": {"invoke_exit": 0, "timed_out": false, "verify_verdict": "PASS_WITH_ISSUES", "elapsed_seconds": 100},
-  "pair": {"invoke_exit": 0, "timed_out": false, "verify_verdict": "NEEDS_WORK", "pair_mode": true, "elapsed_seconds": 200},
+  "pair": {
+    "invoke_exit": 0,
+    "timed_out": false,
+    "verify_verdict": "NEEDS_WORK",
+    "pair_mode": true,
+    "pair_trigger": {"eligible": true, "reasons": ["mode.verify-only"], "skipped_reason": null},
+    "elapsed_seconds": 200
+  },
   "comparison": {"pair_trigger_missed": false, "pair_verdict_lift": true, "pair_internal_verdict_lift": false}
 }
 EOF
@@ -86,6 +114,137 @@ python3 "$GATE" --results-root "$TMP_DIR" --fixtures-root "$FIXTURES_DIR" \
   > "$TMP_DIR/summary-verdicts.out"
 grep -Fq '"verdict": "PASS"' "$TMP_DIR/summary-verdicts.out"
 
+mkdir -p "$TMP_DIR/string-pair-mode/pair"
+cat > "$TMP_DIR/string-pair-mode/pair/input.md" <<'EOF'
+Use /devlyn:resolve --verify-only --spec docs/roadmap/phase-1/F13-string-pair-mode.md.
+EOF
+cat > "$TMP_DIR/string-pair-mode/compare.json" <<'EOF'
+{
+  "solo": {"invoke_exit": 0, "timed_out": false, "verify_verdict": "PASS_WITH_ISSUES", "elapsed_seconds": 100},
+  "pair": {
+    "invoke_exit": 0,
+    "timed_out": false,
+    "verify_verdict": "NEEDS_WORK",
+    "pair_mode": "true",
+    "pair_trigger": {"eligible": true, "reasons": ["mode.verify-only"], "skipped_reason": null},
+    "elapsed_seconds": 200
+  },
+  "comparison": {"pair_trigger_missed": false, "pair_verdict_lift": true, "pair_internal_verdict_lift": false}
+}
+EOF
+mkdir -p "$FIXTURES_DIR/F13-string-pair-mode"
+expect_fail_contains string-pair-mode "pair_mode false" \
+  python3 "$GATE" --results-root "$TMP_DIR" --fixtures-root "$FIXTURES_DIR" \
+    --run-id string-pair-mode --min-runs 1
+
+write_run malformed-pair-trigger F13-malformed-pair-trigger PASS_WITH_ISSUES NEEDS_WORK true
+mkdir -p "$FIXTURES_DIR/F13-malformed-pair-trigger"
+python3 - "$TMP_DIR/malformed-pair-trigger/compare.json" <<'PY'
+import json
+import sys
+path = sys.argv[1]
+with open(path) as f:
+    data = json.load(f)
+data["pair"]["pair_trigger"] = {"eligible": True, "reasons": "forced_pair", "skipped_reason": None}
+with open(path, "w") as f:
+    json.dump(data, f)
+PY
+expect_fail_contains malformed-pair-trigger "pair_trigger.reasons malformed" \
+  python3 "$GATE" --results-root "$TMP_DIR" --fixtures-root "$FIXTURES_DIR" \
+    --run-id malformed-pair-trigger --min-runs 1
+
+write_run unknown-pair-trigger F13-unknown-pair-trigger PASS_WITH_ISSUES NEEDS_WORK true
+mkdir -p "$FIXTURES_DIR/F13-unknown-pair-trigger"
+python3 - "$TMP_DIR/unknown-pair-trigger/compare.json" <<'PY'
+import json
+import sys
+path = sys.argv[1]
+with open(path) as f:
+    data = json.load(f)
+data["pair"]["pair_trigger"] = {"eligible": True, "reasons": ["looks-hard"], "skipped_reason": None}
+with open(path, "w") as f:
+    json.dump(data, f)
+PY
+expect_fail_contains unknown-pair-trigger "pair_trigger reasons missing known trigger reason" \
+  python3 "$GATE" --results-root "$TMP_DIR" --fixtures-root "$FIXTURES_DIR" \
+    --run-id unknown-pair-trigger --min-runs 1
+
+write_run mixed-unknown-pair-trigger F13-mixed-unknown-pair-trigger PASS_WITH_ISSUES NEEDS_WORK true
+mkdir -p "$FIXTURES_DIR/F13-mixed-unknown-pair-trigger"
+python3 - "$TMP_DIR/mixed-unknown-pair-trigger/compare.json" <<'PY'
+import json
+import sys
+path = sys.argv[1]
+with open(path) as f:
+    data = json.load(f)
+data["pair"]["pair_trigger"] = {"eligible": True, "reasons": ["mode.verify-only", "looks-hard"], "skipped_reason": None}
+with open(path, "w") as f:
+    json.dump(data, f)
+PY
+expect_fail_contains mixed-unknown-pair-trigger "pair_trigger reasons contain unknown trigger reason" \
+  python3 "$GATE" --results-root "$TMP_DIR" --fixtures-root "$FIXTURES_DIR" \
+    --run-id mixed-unknown-pair-trigger --min-runs 1
+
+write_run normalized-canonical-pair-trigger F13-normalized-canonical-pair-trigger PASS_WITH_ISSUES NEEDS_WORK true
+mkdir -p "$FIXTURES_DIR/F13-normalized-canonical-pair-trigger"
+python3 - "$TMP_DIR/normalized-canonical-pair-trigger/compare.json" <<'PY'
+import json
+import sys
+path = sys.argv[1]
+with open(path) as f:
+    data = json.load(f)
+data["pair"]["pair_trigger"] = {"eligible": True, "reasons": ["risk high"], "skipped_reason": None}
+with open(path, "w") as f:
+    json.dump(data, f)
+PY
+expect_fail_contains normalized-canonical-pair-trigger "pair_trigger reasons missing known trigger reason" \
+  python3 "$GATE" --results-root "$TMP_DIR" --fixtures-root "$FIXTURES_DIR" \
+    --run-id normalized-canonical-pair-trigger --min-runs 1
+
+write_run historical-only-pair-trigger F13-historical-only-pair-trigger PASS_WITH_ISSUES NEEDS_WORK true
+mkdir -p "$FIXTURES_DIR/F13-historical-only-pair-trigger"
+python3 - "$TMP_DIR/historical-only-pair-trigger/compare.json" <<'PY'
+import json
+import sys
+path = sys.argv[1]
+with open(path) as f:
+    data = json.load(f)
+data["pair"]["pair_trigger"] = {"eligible": True, "reasons": ["risk_profile.high_risk"], "skipped_reason": None}
+with open(path, "w") as f:
+    json.dump(data, f)
+PY
+expect_fail_contains historical-only-pair-trigger "pair_trigger reasons missing canonical trigger reason" \
+  python3 "$GATE" --results-root "$TMP_DIR" --fixtures-root "$FIXTURES_DIR" \
+    --run-id historical-only-pair-trigger --min-runs 1
+
+write_run missing-hypothesis-trigger F13-missing-hypothesis-trigger PASS_WITH_ISSUES NEEDS_WORK true
+mkdir -p "$FIXTURES_DIR/F13-missing-hypothesis-trigger"
+cat > "$FIXTURES_DIR/F13-missing-hypothesis-trigger/spec.md" <<'EOF'
+## Verification
+
+- Solo-headroom hypothesis: `solo_claude` is expected to miss the refund-window defect; observable miss command: `npm test -- --runInBand`.
+EOF
+expect_fail_contains missing-hypothesis-trigger "pair_trigger missing spec.solo_headroom_hypothesis" \
+  python3 "$GATE" --results-root "$TMP_DIR" --fixtures-root "$FIXTURES_DIR" \
+    --run-id missing-hypothesis-trigger --min-runs 1 --require-hypothesis-trigger
+python3 - "$TMP_DIR/missing-hypothesis-trigger/compare.json" <<'PY'
+import json
+import sys
+path = sys.argv[1]
+with open(path) as f:
+    data = json.load(f)
+data["pair"]["pair_trigger"]["reasons"] = [
+    "mode.verify-only",
+    "spec.solo_headroom_hypothesis",
+]
+with open(path, "w") as f:
+    json.dump(data, f)
+PY
+python3 "$GATE" --results-root "$TMP_DIR" --fixtures-root "$FIXTURES_DIR" \
+  --run-id missing-hypothesis-trigger --min-runs 1 --require-hypothesis-trigger \
+  > "$TMP_DIR/hypothesis-trigger-pass.out"
+grep -Fq '"verdict": "PASS"' "$TMP_DIR/hypothesis-trigger-pass.out"
+
 write_run dup-a F12-webhook-raw-body-signature PASS_WITH_ISSUES NEEDS_WORK true
 write_run dup-b F12-webhook-raw-body-signature PASS_WITH_ISSUES NEEDS_WORK true
 expect_fail_contains duplicate-fixture "duplicate fixture_id=F12-webhook-raw-body-signature" \
@@ -138,7 +297,13 @@ EOF
 cat > "$TMP_DIR/missing-elapsed/compare.json" <<'EOF'
 {
   "solo": {"invoke_exit": 0, "timed_out": false, "verify_verdict": "PASS_WITH_ISSUES"},
-  "pair": {"invoke_exit": 0, "timed_out": false, "verify_verdict": "NEEDS_WORK", "pair_mode": true},
+  "pair": {
+    "invoke_exit": 0,
+    "timed_out": false,
+    "verify_verdict": "NEEDS_WORK",
+    "pair_mode": true,
+    "pair_trigger": {"eligible": true, "reasons": ["mode.verify-only"], "skipped_reason": null}
+  },
   "comparison": {
     "pair_trigger_missed": false,
     "pair_verdict_lift": true,
@@ -164,6 +329,117 @@ expect_fail_contains missing-compare "missing compare.json for missing-compare"
   python3 "$GATE" --results-root "$TMP_DIR" --fixtures-root "$FIXTURES_DIR" \
     --run-id missing-compare --min-runs 1
 
+mkdir -p "$TMP_DIR/malformed-compare/pair"
+cat > "$TMP_DIR/malformed-compare/pair/input.md" <<'EOF'
+Use /devlyn:resolve --verify-only --spec docs/roadmap/phase-1/F17-malformed-compare.md.
+EOF
+printf '["not", "a", "dict"]\n' > "$TMP_DIR/malformed-compare/compare.json"
+mkdir -p "$FIXTURES_DIR/F17-malformed-compare"
+expect_fail_contains malformed-compare "malformed compare.json for malformed-compare: expected object" \
+  python3 "$GATE" --results-root "$TMP_DIR" --fixtures-root "$FIXTURES_DIR" \
+    --run-id malformed-compare --min-runs 1
+
+mkdir -p "$TMP_DIR/nan-compare/pair"
+cat > "$TMP_DIR/nan-compare/pair/input.md" <<'EOF'
+Use /devlyn:resolve --verify-only --spec docs/roadmap/phase-1/F17-nan-compare.md.
+EOF
+cat > "$TMP_DIR/nan-compare/compare.json" <<'EOF'
+{
+  "solo": {"invoke_exit": 0, "timed_out": false, "verify_verdict": "PASS_WITH_ISSUES", "elapsed_seconds": 100},
+  "pair": {
+    "invoke_exit": 0,
+    "timed_out": false,
+    "verify_verdict": "NEEDS_WORK",
+    "pair_mode": true,
+    "pair_trigger": {"eligible": true, "reasons": ["mode.verify-only"], "skipped_reason": null},
+    "elapsed_seconds": NaN
+  },
+  "comparison": {"pair_trigger_missed": false, "pair_verdict_lift": true, "pair_internal_verdict_lift": false}
+}
+EOF
+mkdir -p "$FIXTURES_DIR/F17-nan-compare"
+expect_fail_contains nan-compare "malformed compare.json for nan-compare: invalid JSON" \
+  python3 "$GATE" --results-root "$TMP_DIR" --fixtures-root "$FIXTURES_DIR" \
+    --run-id nan-compare --min-runs 1
+
+mkdir -p "$TMP_DIR/malformed-compare-sections/pair"
+cat > "$TMP_DIR/malformed-compare-sections/pair/input.md" <<'EOF'
+Use /devlyn:resolve --verify-only --spec docs/roadmap/phase-1/F17-malformed-compare-sections.md.
+EOF
+cat > "$TMP_DIR/malformed-compare-sections/compare.json" <<'EOF'
+{
+  "solo": ["not", "a", "dict"],
+  "pair": ["not", "a", "dict"],
+  "comparison": ["not", "a", "dict"]
+}
+EOF
+mkdir -p "$FIXTURES_DIR/F17-malformed-compare-sections"
+expect_fail_contains malformed-compare-sections "pair_mode false" \
+  python3 "$GATE" --results-root "$TMP_DIR" --fixtures-root "$FIXTURES_DIR" \
+    --run-id malformed-compare-sections --min-runs 1
+
+mkdir -p "$TMP_DIR/malformed-verdict-fields/pair"
+cat > "$TMP_DIR/malformed-verdict-fields/pair/input.md" <<'EOF'
+Use /devlyn:resolve --verify-only --spec docs/roadmap/phase-1/F17-malformed-verdict-fields.md.
+EOF
+cat > "$TMP_DIR/malformed-verdict-fields/compare.json" <<'EOF'
+{
+  "solo": {"invoke_exit": 0, "timed_out": false, "verify_verdict": ["bad"], "elapsed_seconds": 100},
+  "pair": {
+    "invoke_exit": 0,
+    "timed_out": false,
+    "verify_verdict": ["bad"],
+    "pair_mode": true,
+    "pair_trigger": {"eligible": true, "reasons": ["mode.verify-only"], "skipped_reason": null},
+    "elapsed_seconds": 200
+  },
+  "comparison": {
+    "pair_trigger_missed": false,
+    "pair_verdict_lift": true,
+    "pair_internal_verdict_lift": true,
+    "solo_verdict": ["bad"],
+    "pair_verdict": ["bad"],
+    "pair_primary_verdict": ["bad"],
+    "pair_judge_verdict": ["bad"]
+  }
+}
+EOF
+mkdir -p "$FIXTURES_DIR/F17-malformed-verdict-fields"
+expect_fail_contains malformed-verdict-fields "pair verdict missing or malformed" \
+  python3 "$GATE" --results-root "$TMP_DIR" --fixtures-root "$FIXTURES_DIR" \
+    --run-id malformed-verdict-fields --min-runs 1
+
+mkdir -p "$TMP_DIR/malformed-elapsed-fields/pair"
+cat > "$TMP_DIR/malformed-elapsed-fields/pair/input.md" <<'EOF'
+Use /devlyn:resolve --verify-only --spec docs/roadmap/phase-1/F17-malformed-elapsed-fields.md.
+EOF
+cat > "$TMP_DIR/malformed-elapsed-fields/compare.json" <<'EOF'
+{
+  "solo": {"invoke_exit": 0, "timed_out": false, "verify_verdict": "PASS_WITH_ISSUES", "elapsed_seconds": true},
+  "pair": {
+    "invoke_exit": 0,
+    "timed_out": false,
+    "verify_verdict": "NEEDS_WORK",
+    "pair_mode": true,
+    "pair_trigger": {"eligible": true, "reasons": ["mode.verify-only"], "skipped_reason": null},
+    "elapsed_seconds": false
+  },
+  "comparison": {
+    "pair_trigger_missed": false,
+    "pair_verdict_lift": true,
+    "pair_internal_verdict_lift": false,
+    "solo_verdict": "PASS_WITH_ISSUES",
+    "pair_verdict": "NEEDS_WORK",
+    "pair_primary_verdict": "NEEDS_WORK",
+    "pair_judge_verdict": "NEEDS_WORK"
+  }
+}
+EOF
+mkdir -p "$FIXTURES_DIR/F17-malformed-elapsed-fields"
+expect_fail_contains malformed-elapsed-fields "pair/solo wall ratio missing" \
+  python3 "$GATE" --results-root "$TMP_DIR" --fixtures-root "$FIXTURES_DIR" \
+    --run-id malformed-elapsed-fields --min-runs 1 --max-pair-solo-wall-ratio 3
+
 mkdir -p "$TMP_DIR/provider-limit/pair"
 cat > "$TMP_DIR/provider-limit/pair/input.md" <<'EOF'
 Use /devlyn:resolve --verify-only --spec docs/roadmap/phase-1/F18-provider-limit.md.
@@ -189,4 +465,53 @@ expect_fail_contains provider-limit "pair provider limit" \
   python3 "$GATE" --results-root "$TMP_DIR" --fixtures-root "$FIXTURES_DIR" \
     --run-id provider-limit --min-runs 1
 
+write_run dirty-pair-env F19-dirty-pair-env PASS_WITH_ISSUES NEEDS_WORK true
+mkdir -p "$FIXTURES_DIR/F19-dirty-pair-env"
+python3 - "$TMP_DIR/dirty-pair-env/compare.json" <<'PY'
+import json
+import sys
+path = sys.argv[1]
+with open(path) as f:
+    data = json.load(f)
+data["pair"]["environment_contamination"] = True
+with open(path, "w") as f:
+    json.dump(data, f)
+PY
+expect_fail_contains dirty-pair-env "pair environment contamination" \
+  python3 "$GATE" --results-root "$TMP_DIR" --fixtures-root "$FIXTURES_DIR" \
+    --run-id dirty-pair-env --min-runs 1
+
+write_run dirty-solo-disqualifier F20-dirty-solo-disqualifier PASS_WITH_ISSUES NEEDS_WORK true
+mkdir -p "$FIXTURES_DIR/F20-dirty-solo-disqualifier"
+python3 - "$TMP_DIR/dirty-solo-disqualifier/compare.json" <<'PY'
+import json
+import sys
+path = sys.argv[1]
+with open(path) as f:
+    data = json.load(f)
+data["solo"]["disqualifier"] = True
+with open(path, "w") as f:
+    json.dump(data, f)
+PY
+expect_fail_contains dirty-solo-disqualifier "solo disqualifier" \
+  python3 "$GATE" --results-root "$TMP_DIR" --fixtures-root "$FIXTURES_DIR" \
+    --run-id dirty-solo-disqualifier --min-runs 1
+
+write_run dirty-pair-invoke F21-dirty-pair-invoke PASS_WITH_ISSUES NEEDS_WORK true
+mkdir -p "$FIXTURES_DIR/F21-dirty-pair-invoke"
+python3 - "$TMP_DIR/dirty-pair-invoke/compare.json" <<'PY'
+import json
+import sys
+path = sys.argv[1]
+with open(path) as f:
+    data = json.load(f)
+data["pair"]["invoke_failure"] = True
+data["pair"]["invoke_failure_reason"] = "plugin_contamination"
+with open(path, "w") as f:
+    json.dump(data, f)
+PY
+expect_fail_contains dirty-pair-invoke "pair invoke failure (plugin_contamination)" \
+  python3 "$GATE" --results-root "$TMP_DIR" --fixtures-root "$FIXTURES_DIR" \
+    --run-id dirty-pair-invoke --min-runs 1
+
 echo "✓ test-frozen-verify-gate"