devlyn-cli 2.0.0 → 2.2.0

package/benchmark/auto-resolve/scripts/fetch-swebench-instances.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+"""Fetch SWE-bench instances as JSONL without Hugging Face Python deps."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import urllib.parse
+import urllib.request
+from pathlib import Path
+from typing import Any
+
+
+DATASETS = {
+    "lite": "princeton-nlp/SWE-bench_Lite",
+    "verified": "princeton-nlp/SWE-bench_Verified",
+    "full": "princeton-nlp/SWE-bench",
+}
+
+
+def fetch_rows(dataset: str, split: str, offset: int, length: int) -> dict[str, Any]:
+    params = urllib.parse.urlencode(
+        {
+            "dataset": dataset,
+            "config": "default",
+            "split": split,
+            "offset": offset,
+            "length": length,
+        }
+    )
+    url = f"https://datasets-server.huggingface.co/rows?{params}"
+    with urllib.request.urlopen(url, timeout=60) as response:
+        return json.load(response)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--dataset", choices=sorted(DATASETS), default="lite")
+    parser.add_argument("--dataset-id", help="Override the Hugging Face dataset id.")
+    parser.add_argument("--split", default="test")
+    parser.add_argument("--limit", type=int, help="Fetch at most N rows.")
+    parser.add_argument("--page-size", type=int, default=100)
+    parser.add_argument("--instance-id", action="append", help="Keep only these instance ids.")
+    parser.add_argument("--out", required=True, type=Path)
+    args = parser.parse_args()
+
+    if args.page_size <= 0:
+        raise ValueError("--page-size must be > 0")
+    dataset = args.dataset_id or DATASETS[args.dataset]
+    keep = set(args.instance_id or [])
+    rows: list[dict[str, Any]] = []
+    offset = 0
+    total: int | None = None
+
+    while True:
+        remaining = args.page_size
+        if args.limit is not None:
+            remaining = min(remaining, max(args.limit - len(rows), 0))
+            if remaining == 0:
+                break
+        page = fetch_rows(dataset, args.split, offset, remaining)
+        if total is None:
+            total = int(page.get("num_rows_total") or 0)
+        page_rows = page.get("rows") or []
+        if not page_rows:
+            break
+        for wrapper in page_rows:
+            row = wrapper.get("row")
+            if not isinstance(row, dict):
+                continue
+            instance_id = row.get("instance_id")
+            if keep and instance_id not in keep:
+                continue
+            rows.append(row)
+            if args.limit is not None and len(rows) >= args.limit:
+                break
+        offset += len(page_rows)
+        if offset >= total:
+            break
+
+    if keep:
+        found = {row.get("instance_id") for row in rows}
+        missing = sorted(keep - found)
+        if missing:
+            raise ValueError(f"requested instance ids not found in fetched split: {', '.join(missing)}")
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    with args.out.open("w", encoding="utf8") as f:
+        for row in rows:
+            f.write(json.dumps(row, ensure_ascii=False) + "\n")
+    print(
+        json.dumps(
+            {
+                "dataset": dataset,
+                "split": args.split,
+                "rows_written": len(rows),
+                "rows_total": total,
+                "out": str(args.out),
+            },
+            indent=2,
+        )
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except Exception as exc:
+        print(f"fetch-swebench-instances: {exc}", file=sys.stderr)
+        raise SystemExit(1)

package/benchmark/auto-resolve/scripts/frozen-verify-gate.py

@@ -0,0 +1,289 @@
+#!/usr/bin/env python3
+"""Gate frozen VERIFY solo-vs-pair evidence.
+
+This gate is intentionally narrower than headroom-gate.py. It does not claim
+full-pipeline pair superiority. It verifies the leak-free thing we can measure:
+given a fixed external diff, gated pair VERIFY fires and contributes a stricter
+verdict-binding result. That can be either stricter than the separate solo arm
+or stricter than the pair run's own primary judge, which avoids stochastic
+solo-vs-pair confounding. Passing evidence must come from distinct fixture ids
+with runner input metadata present.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+
+VERDICT_RANK = {
+    "PASS": 0,
+    "PASS_WITH_ISSUES": 1,
+    "NEEDS_WORK": 2,
+    "BLOCKED": 3,
+}
+
+
+def load_compare(results_root: Path, run_id: str) -> dict[str, Any]:
+    compare_path = results_root / run_id / "compare.json"
+    if not compare_path.exists():
+        raise FileNotFoundError(f"missing compare.json for {run_id}: {compare_path}")
+    with compare_path.open() as f:
+        return json.load(f)
+
+
+def rank(verdict: str | None) -> int:
+    return VERDICT_RANK.get(verdict or "", -1)
+
+
+def elapsed_ratio(pair_elapsed: Any, solo_elapsed: Any) -> float | None:
+    if not isinstance(pair_elapsed, (int, float)) or not isinstance(solo_elapsed, (int, float)):
+        return None
+    if solo_elapsed <= 0:
+        return None
+    return pair_elapsed / solo_elapsed
+
+
+def infer_fixture_id(results_root: Path, run_id: str) -> str | None:
+    run_root = results_root / run_id
+    for arm in ("pair", "solo"):
+        input_path = run_root / arm / "input.md"
+        if not input_path.exists():
+            continue
+        match = re.search(r"docs/roadmap/phase-1/([^`\s]+)\.md", input_path.read_text())
+        if match:
+            return match.group(1)
+    return None
+
+
+def transcript_failure_reason(results_root: Path, run_id: str, arm: str) -> str | None:
+    transcript_path = results_root / run_id / arm / "transcript.txt"
+    if not transcript_path.is_file():
+        return None
+    transcript = transcript_path.read_text(encoding="utf8", errors="replace")
+    if "You've hit your limit" in transcript:
+        return "provider_limit"
+    return None
+
+
+def evaluate_run(
+    results_root: Path,
+    fixtures_root: Path,
+    run_id: str,
+    max_pair_solo_wall_ratio: float | None,
+) -> dict[str, Any]:
+    try:
+        compare = load_compare(results_root, run_id)
+    except FileNotFoundError as exc:
+        fixture_id = infer_fixture_id(results_root, run_id)
+        return {
+            "run_id": run_id,
+            "fixture_id": fixture_id,
+            "status": "FAIL",
+            "failures": [str(exc)],
+            "solo_verdict": None,
+            "pair_verdict": None,
+            "pair_mode": False,
+            "pair_trigger_missed": False,
+            "pair_verdict_lift": False,
+            "pair_internal_verdict_lift": False,
+            "pair_primary_verdict": None,
+            "pair_judge_verdict": None,
+            "solo_elapsed_seconds": None,
+            "pair_elapsed_seconds": None,
+            "pair_solo_wall_ratio": None,
+            "pair_severity_counts": {},
+        }
+    solo = compare.get("solo") or {}
+    pair = compare.get("pair") or {}
+    comparison = compare.get("comparison") or {}
+    solo_failure_reason = solo.get("invoke_failure_reason") or transcript_failure_reason(
+        results_root, run_id, "solo"
+    )
+    pair_failure_reason = pair.get("invoke_failure_reason") or transcript_failure_reason(
+        results_root, run_id, "pair"
+    )
+
+    failures: list[str] = []
+    if solo.get("timed_out"):
+        failures.append("solo timed out")
+    if pair.get("timed_out"):
+        failures.append("pair timed out")
+    if solo_failure_reason == "provider_limit":
+        failures.append("solo provider limit")
+    if pair_failure_reason == "provider_limit":
+        failures.append("pair provider limit")
+    if solo.get("invoke_exit") != 0:
+        failures.append(f"solo invoke_exit={solo.get('invoke_exit')}")
+    if pair.get("invoke_exit") != 0:
+        failures.append(f"pair invoke_exit={pair.get('invoke_exit')}")
+    if not pair.get("pair_mode"):
+        failures.append("pair_mode false")
+    if comparison.get("pair_trigger_missed"):
+        failures.append("pair trigger missed")
+    external_lift = bool(comparison.get("pair_verdict_lift"))
+    internal_lift = bool(comparison.get("pair_internal_verdict_lift"))
+    if not (external_lift or internal_lift):
+        failures.append("pair verdict lift false")
+
+    solo_verdict = (
+        comparison.get("solo_verdict")
+        or solo.get("verify_verdict")
+        or solo.get("terminal_verdict")
+    )
+    pair_verdict = (
+        comparison.get("pair_verdict")
+        or pair.get("verify_verdict")
+        or pair.get("terminal_verdict")
+    )
+    pair_primary_verdict = comparison.get("pair_primary_verdict")
+    pair_judge_verdict = comparison.get("pair_judge_verdict")
+    if external_lift and rank(pair_verdict) <= rank(solo_verdict):
+        failures.append(f"pair verdict {pair_verdict} not stricter than solo {solo_verdict}")
+    if internal_lift and rank(pair_judge_verdict) <= rank(pair_primary_verdict):
+        failures.append(
+            f"pair_judge verdict {pair_judge_verdict} not stricter than primary {pair_primary_verdict}"
+        )
+    if rank(pair_verdict) < VERDICT_RANK["NEEDS_WORK"]:
+        failures.append(f"pair verdict {pair_verdict} is not verdict-binding")
+    pair_elapsed = pair.get("elapsed_seconds")
+    solo_elapsed = solo.get("elapsed_seconds")
+    wall_ratio = elapsed_ratio(pair_elapsed, solo_elapsed)
+    if max_pair_solo_wall_ratio is not None:
+        if wall_ratio is None:
+            failures.append("pair/solo wall ratio missing")
+        elif wall_ratio > max_pair_solo_wall_ratio:
+            failures.append(
+                f"pair/solo wall ratio {wall_ratio:.2f} exceeds {max_pair_solo_wall_ratio:.2f}"
+            )
+    fixture_id = infer_fixture_id(results_root, run_id)
+    if not fixture_id:
+        failures.append("fixture_id missing")
+    elif not (fixtures_root / fixture_id).is_dir():
+        failures.append(f"fixture_id not found: {fixture_id}")
+
+    return {
+        "run_id": run_id,
+        "fixture_id": fixture_id,
+        "status": "PASS" if not failures else "FAIL",
+        "failures": failures,
+        "solo_verdict": solo_verdict,
+        "pair_verdict": pair_verdict,
+        "pair_mode": bool(pair.get("pair_mode")),
+        "pair_trigger_missed": bool(comparison.get("pair_trigger_missed")),
+        "pair_verdict_lift": external_lift,
+        "pair_internal_verdict_lift": internal_lift,
+        "pair_primary_verdict": pair_primary_verdict,
+        "pair_judge_verdict": pair_judge_verdict,
+        "solo_elapsed_seconds": solo_elapsed,
+        "pair_elapsed_seconds": pair_elapsed,
+        "pair_solo_wall_ratio": wall_ratio,
+        "solo_failure_reason": solo_failure_reason,
+        "pair_failure_reason": pair_failure_reason,
+        "pair_severity_counts": pair.get("severity_counts") or {},
+    }
+
+
+def format_ratio(value: Any) -> str:
+    if isinstance(value, (int, float)):
+        return f"{value:.2f}x"
+    return "n/a"
+
+
+def write_markdown(path: Path, report: dict[str, Any]) -> None:
+    lines = [
+        f"# Frozen VERIFY Gate — {report['run_ids_label']}",
+        "",
+        f"Verdict: **{report['verdict']}**",
+        "",
+        "Rule: every supplied run must be clean, each run must cover a distinct fixture, "
+        "gated pair VERIFY must fire, and pair must contribute a stricter "
+        "verdict-binding result than either the separate solo arm or the pair "
+        "run's own primary judge.",
+        "",
+        f"Minimum passing runs: {report['min_runs']}",
+        f"Max pair/solo wall ratio: {format_ratio(report.get('max_pair_solo_wall_ratio'))}",
+        f"Average pair/solo wall ratio: {format_ratio(report.get('avg_pair_solo_wall_ratio'))}",
+        "",
+        "| Run | Fixture | Solo | Pair | Pair mode | Wall ratio | External lift | Internal lift | Status | Reason |",
+        "|---|---|---|---|---|---|---|---|---|---|",
+    ]
+    for row in report["rows"]:
+        reason = "; ".join(row["failures"]) if row["failures"] else "ok"
+        lines.append(
+            f"| {row['run_id']} | {row.get('fixture_id') or 'unknown'} | "
+            f"{row['solo_verdict']} | {row['pair_verdict']} | "
+            f"{str(row['pair_mode']).lower()} | {format_ratio(row.get('pair_solo_wall_ratio'))} | "
+            f"{str(row['pair_verdict_lift']).lower()} | "
+            f"{str(row['pair_internal_verdict_lift']).lower()} | "
+            f"{row['status']} | {reason} |"
+        )
+    lines.append("")
+    path.write_text("\n".join(lines), encoding="utf8")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--results-root", default="benchmark/auto-resolve/results")
+    parser.add_argument("--fixtures-root", default="benchmark/auto-resolve/fixtures")
+    parser.add_argument("--run-id", action="append", required=True)
+    parser.add_argument("--min-runs", type=int, default=2)
+    parser.add_argument(
+        "--max-pair-solo-wall-ratio",
+        type=float,
+        help="Optional efficiency cap. When set, every run must include elapsed_seconds and pair/solo wall ratio must not exceed this value.",
+    )
+    parser.add_argument("--out-json")
+    parser.add_argument("--out-md")
+    args = parser.parse_args()
+
+    results_root = Path(args.results_root)
+    fixtures_root = Path(args.fixtures_root)
+    rows = [
+        evaluate_run(results_root, fixtures_root, run_id, args.max_pair_solo_wall_ratio)
+        for run_id in args.run_id
+    ]
+    fixture_counts: dict[str, int] = {}
+    for row in rows:
+        fixture_id = row.get("fixture_id")
+        if fixture_id:
+            fixture_counts[fixture_id] = fixture_counts.get(fixture_id, 0) + 1
+    for row in rows:
+        fixture_id = row.get("fixture_id")
+        if fixture_id and fixture_counts.get(fixture_id, 0) > 1:
+            row["failures"].append(f"duplicate fixture_id={fixture_id}")
+            row["status"] = "FAIL"
+    passing = [row for row in rows if row["status"] == "PASS"]
+    verdict = "PASS" if len(passing) >= args.min_runs and len(passing) == len(rows) else "FAIL"
+    ratios = [
+        row["pair_solo_wall_ratio"]
+        for row in rows
+        if isinstance(row.get("pair_solo_wall_ratio"), (int, float))
+    ]
+
+    report = {
+        "run_ids_label": ", ".join(args.run_id),
+        "rule": "clean frozen diff; distinct fixture per run; gated pair VERIFY fires; pair contributes a stricter verdict-binding result; optional pair/solo wall-ratio cap",
+        "min_runs": args.min_runs,
+        "max_pair_solo_wall_ratio": args.max_pair_solo_wall_ratio,
+        "avg_pair_solo_wall_ratio": (sum(ratios) / len(ratios)) if ratios else None,
+        "verdict": verdict,
+        "runs_total": len(rows),
+        "runs_passed": len(passing),
+        "rows": rows,
+    }
+
+    if args.out_json:
+        Path(args.out_json).write_text(json.dumps(report, indent=2) + "\n", encoding="utf8")
+    if args.out_md:
+        write_markdown(Path(args.out_md), report)
+
+    print(json.dumps(report, indent=2))
+    return 0 if verdict == "PASS" else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py

@@ -0,0 +1,250 @@
+#!/usr/bin/env python3
+"""Gate full-pipeline L2/pair evidence against L1 solo.
+
+This is stricter than headroom-gate.py. Headroom only says a candidate set is
+worth measuring. This gate says the measured L2 arm is usable evidence:
+bare and solo leave headroom, l2_gated is clean, gated pair actually fired, and
+the blind judge scores l2_gated materially above solo_claude.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import pathlib
+import sys
+from typing import Any
+
+
+def load_json(path: pathlib.Path) -> dict[str, Any] | None:
+    if not path.is_file():
+        return None
+    return json.loads(path.read_text())
+
+
+def score_for(judge: dict[str, Any], arm: str) -> int | None:
+    value = (judge.get("scores_by_arm") or {}).get(arm)
+    return value if isinstance(value, int) else None
+
+
+def clean_failures(fixture_dir: pathlib.Path, judge: dict[str, Any], arm: str) -> list[str]:
+    failures: list[str] = []
+    result = load_json(fixture_dir / arm / "result.json")
+    verify = load_json(fixture_dir / arm / "verify.json")
+    if result is None:
+        failures.append(f"{arm} result.json missing")
+    if verify is None:
+        failures.append(f"{arm} verify.json missing")
+
+    dq_by_arm = judge.get("disqualifiers_by_arm") or {}
+    if bool((dq_by_arm.get(arm) or {}).get("disqualifier")):
+        failures.append(f"{arm} judge disqualifier")
+    if result is not None:
+        if bool(result.get("disqualifier")):
+            failures.append(f"{arm} result disqualifier")
+        if bool(result.get("timed_out")):
+            failures.append(f"{arm} timed out")
+        if bool(result.get("invoke_failure")):
+            reason = result.get("invoke_failure_reason")
+            if isinstance(reason, str) and reason:
+                failures.append(f"{arm} invoke failure ({reason})")
+            else:
+                failures.append(f"{arm} invoke failure")
+    if verify is not None and bool(verify.get("disqualifier")):
+        failures.append(f"{arm} verify disqualifier")
+    return failures
+
+
+def elapsed_ratio(pair_result: dict[str, Any] | None, solo_result: dict[str, Any] | None) -> float | None:
+    if pair_result is None or solo_result is None:
+        return None
+    pair_elapsed = pair_result.get("elapsed_seconds")
+    solo_elapsed = solo_result.get("elapsed_seconds")
+    if not isinstance(pair_elapsed, (int, float)) or not isinstance(solo_elapsed, (int, float)):
+        return None
+    if solo_elapsed <= 0:
+        return None
+    return pair_elapsed / solo_elapsed
+
+
+def provider_limited(result: dict[str, Any] | None) -> bool:
+    return result is not None and result.get("invoke_failure_reason") == "provider_limit"
+
+
+def evaluate_fixture(
+    fixture_dir: pathlib.Path,
+    *,
+    pair_arm: str,
+    bare_max: int,
+    solo_max: int,
+    min_pair_margin: int,
+    max_pair_solo_wall_ratio: float | None,
+) -> dict[str, Any]:
+    judge = load_json(fixture_dir / "judge.json")
+    if judge is None:
+        return {
+            "fixture": fixture_dir.name,
+            "status": "FAIL",
+            "reason": "judge.json missing",
+        }
+
+    bare = score_for(judge, "bare")
+    solo = score_for(judge, "solo_claude")
+    pair = score_for(judge, pair_arm)
+    solo_result = load_json(fixture_dir / "solo_claude" / "result.json")
+    pair_result = load_json(fixture_dir / pair_arm / "result.json")
+    ratio = elapsed_ratio(pair_result, solo_result)
+    pair_provider_limited = provider_limited(pair_result)
+    if pair_provider_limited:
+        ratio = None
+
+    reasons: list[str] = []
+    if bare is None:
+        reasons.append("bare score missing")
+    elif bare > bare_max:
+        reasons.append(f"bare score {bare} > {bare_max}")
+    if solo is None:
+        reasons.append("solo_claude score missing")
+    elif solo > solo_max:
+        reasons.append(f"solo_claude score {solo} > {solo_max}")
+    if pair_provider_limited:
+        pass
+    elif pair is None:
+        reasons.append(f"{pair_arm} score missing")
+    elif solo is not None and pair - solo < min_pair_margin:
+        reasons.append(f"{pair_arm} margin {pair - solo:+d} < +{min_pair_margin}")
+
+    reasons.extend(clean_failures(fixture_dir, judge, "bare"))
+    reasons.extend(clean_failures(fixture_dir, judge, "solo_claude"))
+    reasons.extend(clean_failures(fixture_dir, judge, pair_arm))
+
+    pair_mode = None if pair_result is None else pair_result.get("pair_mode")
+    if pair_mode is not True and not pair_provider_limited:
+        reasons.append(f"{pair_arm} pair_mode not true")
+
+    if max_pair_solo_wall_ratio is not None and not pair_provider_limited:
+        if ratio is None:
+            reasons.append("pair/solo wall ratio missing")
+        elif ratio > max_pair_solo_wall_ratio:
+            reasons.append(f"pair/solo wall ratio {ratio:.2f} > {max_pair_solo_wall_ratio:.2f}")
+
+    return {
+        "fixture": fixture_dir.name,
+        "status": "PASS" if not reasons else "FAIL",
+        "bare_score": bare,
+        "solo_score": solo,
+        "pair_score": pair,
+        "pair_margin": (
+            None if pair_provider_limited
+            else pair - solo if isinstance(pair, int) and isinstance(solo, int)
+            else None
+        ),
+        "pair_mode": pair_mode,
+        "pair_solo_wall_ratio": ratio,
+        "reason": "; ".join(reasons),
+    }
+
+
+def fmt_ratio(value: Any) -> str:
+    return f"{value:.2f}x" if isinstance(value, (int, float)) else "n/a"
+
+
+def write_md(path: pathlib.Path, report: dict[str, Any]) -> None:
+    lines = [
+        f"# Full-Pipeline Pair Gate - {report['run_id']}",
+        "",
+        f"Verdict: **{report['verdict']}**",
+        "",
+        f"Rule: at least {report['min_fixtures']} fixtures; bare <= {report['bare_max']}; "
+        f"solo_claude <= {report['solo_max']}; {report['pair_arm']} clean; pair_mode true; "
+        f"{report['pair_arm']} - solo_claude >= {report['min_pair_margin']}.",
+        f"Max pair/solo wall ratio: {fmt_ratio(report['max_pair_solo_wall_ratio'])}",
+        f"Average pair/solo wall ratio: {fmt_ratio(report['avg_pair_solo_wall_ratio'])}",
+        "",
+        "| Fixture | Bare | Solo | Pair | Margin | Pair mode | Wall ratio | Status | Reason |",
+        "|---|---:|---:|---:|---:|---|---:|---|---|",
+    ]
+    for row in report["rows"]:
+        margin = row.get("pair_margin")
+        margin_text = f"{margin:+d}" if isinstance(margin, int) else "n/a"
+        lines.append(
+            f"| {row['fixture']} | {row.get('bare_score')} | {row.get('solo_score')} | "
+            f"{row.get('pair_score')} | {margin_text} | {str(row.get('pair_mode')).lower()} | "
+            f"{fmt_ratio(row.get('pair_solo_wall_ratio'))} | {row['status']} | {row.get('reason', '')} |"
+        )
+    lines.append("")
+    path.write_text("\n".join(lines), encoding="utf8")
+
+
+def positive_float(value: str) -> float:
+    parsed = float(value)
+    if parsed <= 0:
+        raise argparse.ArgumentTypeError("value must be > 0")
+    return parsed
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--run-id", required=True)
+    parser.add_argument("--results-root", default="benchmark/auto-resolve/results", type=pathlib.Path)
+    parser.add_argument("--bare-max", type=int, default=60)
+    parser.add_argument("--solo-max", type=int, default=80)
+    parser.add_argument("--min-pair-margin", type=int, default=5)
+    parser.add_argument("--min-fixtures", type=int, default=2)
+    parser.add_argument("--pair-arm", default="l2_gated")
+    parser.add_argument("--max-pair-solo-wall-ratio", type=positive_float)
+    parser.add_argument("--out-json", type=pathlib.Path)
+    parser.add_argument("--out-md", type=pathlib.Path)
+    args = parser.parse_args()
+
+    run_root = args.results_root / args.run_id
+    if not run_root.is_dir():
+        print(f"no results dir: {run_root}", file=sys.stderr)
+        return 2
+
+    rows = [
+        evaluate_fixture(
+            fixture_dir,
+            pair_arm=args.pair_arm,
+            bare_max=args.bare_max,
+            solo_max=args.solo_max,
+            min_pair_margin=args.min_pair_margin,
+            max_pair_solo_wall_ratio=args.max_pair_solo_wall_ratio,
+        )
+        for fixture_dir in sorted(p for p in run_root.iterdir() if p.is_dir())
+    ]
+    pass_count = sum(1 for row in rows if row["status"] == "PASS")
+    fixture_count_ok = len(rows) >= args.min_fixtures
+    verdict = "PASS" if rows and fixture_count_ok and pass_count == len(rows) else "FAIL"
+    ratios = [
+        row["pair_solo_wall_ratio"]
+        for row in rows
+        if isinstance(row.get("pair_solo_wall_ratio"), (int, float))
+    ]
+    report = {
+        "run_id": args.run_id,
+        "rule": "headroom candidates only; l2_gated must be clean, pair_mode true, and beat solo_claude by the configured margin",
+        "verdict": verdict,
+        "fixtures_total": len(rows),
+        "fixtures_passed": pass_count,
+        "min_fixtures": args.min_fixtures,
+        "fixture_count_ok": fixture_count_ok,
+        "bare_max": args.bare_max,
+        "solo_max": args.solo_max,
+        "min_pair_margin": args.min_pair_margin,
+        "pair_arm": args.pair_arm,
+        "max_pair_solo_wall_ratio": args.max_pair_solo_wall_ratio,
+        "avg_pair_solo_wall_ratio": (sum(ratios) / len(ratios)) if ratios else None,
+        "rows": rows,
+    }
+
+    if args.out_json:
+        args.out_json.write_text(json.dumps(report, indent=2) + "\n", encoding="utf8")
+    if args.out_md:
+        write_md(args.out_md, report)
+    else:
+        print(json.dumps(report, indent=2))
+    return 0 if verdict == "PASS" else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())