devlyn-cli 2.1.0 → 2.2.1

package/benchmark/auto-resolve/scripts/headroom-gate.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+"""Headroom gate for candidate L2/pair fixtures.
+
+Pair lift is not measurable when bare and solo already score near the ceiling.
+This gate checks the precondition recorded in HANDOFF.md: before an L2 pair
+measurement is pre-registered, candidate fixtures must leave enough room for
+pair to improve the outcome.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import pathlib
+import sys
+
+
+def load_json(path: pathlib.Path) -> dict | None:
+    if not path.is_file():
+        return None
+    return json.loads(path.read_text())
+
+
+def score_for(judge: dict, arm: str) -> int | None:
+    scores = judge.get("scores_by_arm") or {}
+    value = scores.get(arm)
+    return value if isinstance(value, int) else None
+
+
+def arm_clean_failures(fixture_dir: pathlib.Path, judge: dict, arm: str) -> list[str]:
+    failures: list[str] = []
+    result = load_json(fixture_dir / arm / "result.json")
+    verify = load_json(fixture_dir / arm / "verify.json")
+    if result is None:
+        failures.append(f"{arm} result.json missing")
+    if verify is None:
+        failures.append(f"{arm} verify.json missing")
+    dq_by_arm = judge.get("disqualifiers_by_arm") or {}
+    if bool((dq_by_arm.get(arm) or {}).get("disqualifier")):
+        failures.append(f"{arm} judge disqualifier")
+    if result is not None:
+        if bool(result.get("disqualifier")):
+            failures.append(f"{arm} result disqualifier")
+        if bool(result.get("timed_out")):
+            failures.append(f"{arm} timed out")
+        if bool(result.get("invoke_failure")):
+            failures.append(f"{arm} invoke failure")
+    if verify is not None and bool(verify.get("disqualifier")):
+        failures.append(f"{arm} verify disqualifier")
+    return failures
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--run-id", required=True)
+    parser.add_argument("--results-root", default="benchmark/auto-resolve/results")
+    parser.add_argument("--bare-max", type=int, default=60)
+    parser.add_argument("--solo-max", type=int, default=80)
+    parser.add_argument("--min-fixtures", type=int, default=2)
+    parser.add_argument("--out-json", default=None)
+    parser.add_argument("--out-md", default=None)
+    args = parser.parse_args()
+
+    res_root = pathlib.Path(args.results_root) / args.run_id
+    if not res_root.is_dir():
+        print(f"no results dir: {res_root}", file=sys.stderr)
+        return 2
+
+    rows = []
+    for fixture_dir in sorted(p for p in res_root.iterdir() if p.is_dir()):
+        judge = load_json(fixture_dir / "judge.json")
+        if judge is None:
+            rows.append({
+                "fixture": fixture_dir.name,
+                "status": "MISSING_JUDGE",
+                "reason": "judge.json missing",
+            })
+            continue
+        bare = score_for(judge, "bare")
+        solo = score_for(judge, "solo_claude")
+        bare_clean_failures = arm_clean_failures(fixture_dir, judge, "bare")
+        solo_clean_failures = arm_clean_failures(fixture_dir, judge, "solo_claude")
+        bare_ok = bare is not None and bare <= args.bare_max and not bare_clean_failures
+        solo_ok = solo is not None and solo <= args.solo_max and not solo_clean_failures
+        status = "PASS" if bare_ok and solo_ok else "FAIL"
+        reasons = []
+        if bare is None:
+            reasons.append("bare score missing")
+        elif bare > args.bare_max:
+            reasons.append(f"bare score {bare} > {args.bare_max}")
+        if solo is None:
+            reasons.append("solo_claude score missing")
+        elif solo > args.solo_max:
+            reasons.append(f"solo_claude score {solo} > {args.solo_max}")
+        reasons.extend(bare_clean_failures)
+        reasons.extend(solo_clean_failures)
+        rows.append({
+            "fixture": fixture_dir.name,
+            "status": status,
+            "bare_score": bare,
+            "solo_score": solo,
+            "reason": "; ".join(reasons) if reasons else "",
+        })
+
+    pass_count = sum(1 for row in rows if row["status"] == "PASS")
+    fixture_count_ok = len(rows) >= args.min_fixtures
+    verdict = "PASS" if pass_count == len(rows) and rows and fixture_count_ok else "FAIL"
+    payload = {
+        "run_id": args.run_id,
+        "rule": f"at least {args.min_fixtures} candidate fixtures; each must satisfy bare <= {args.bare_max} and solo_claude <= {args.solo_max}, with both arms clean",
+        "verdict": verdict,
+        "fixtures_total": len(rows),
+        "fixtures_passed": pass_count,
+        "min_fixtures": args.min_fixtures,
+        "fixture_count_ok": fixture_count_ok,
+        "rows": rows,
+    }
+
+    if args.out_json:
+        pathlib.Path(args.out_json).write_text(json.dumps(payload, indent=2) + "\n")
+
+    lines = [
+        f"# Headroom Gate — {args.run_id}",
+        "",
+        f"Verdict: **{verdict}**",
+        "",
+        f"Rule: at least {args.min_fixtures} fixtures; bare <= {args.bare_max}, "
+        f"solo_claude <= {args.solo_max}, both arms clean.",
+        "",
+        "| Fixture | Bare | Solo | Status | Reason |",
+        "|---|---:|---:|---|---|",
+    ]
+    for row in rows:
+        lines.append(
+            f"| {row['fixture']} | {row.get('bare_score')} | {row.get('solo_score')} | "
+            f"{row['status']} | {row.get('reason', '')} |"
+        )
+    report = "\n".join(lines) + "\n"
+    if args.out_md:
+        pathlib.Path(args.out_md).write_text(report)
+    else:
+        print(report)
+
+    return 0 if verdict == "PASS" else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/benchmark/auto-resolve/scripts/judge.sh

@@ -41,11 +41,13 @@ RES_DIR="$BENCH_ROOT/results/$RUN_ID/$FIXTURE"
 # date iter-0019.
 ARMS_PRESENT=()
 # iter-0033c: l2_gated/l2_forced added for NEW L2 vs NEW L1 measurement.
+# iter-0037: l2_risk_probes adds bounded visible-verification probes before
+# IMPLEMENT; judge treats it as another blind arm when artifacts exist.
 # Slot count is still A/B/C max 3 — pair-eligible iter-0033c fixtures supply
 # {solo_claude, l2_gated, l2_forced}; non-pair-eligible fixtures supply
 # {solo_claude, l2_gated}. The blind-shuffle slot mapping below already
 # tolerates arbitrary ARMS_PRESENT counts ≥2.
-for arm in variant solo_claude bare l2_gated l2_forced; do
+for arm in variant solo_claude bare l2_gated l2_risk_probes l2_forced; do
   if [ -f "$RES_DIR/$arm/diff.patch" ] && [ -f "$RES_DIR/$arm/verify.json" ]; then
     ARMS_PRESENT+=("$arm")
   fi
@@ -216,8 +218,34 @@ PY
 # traceable. Run from a clean temp CWD so the judge can't peek at project
 # files that would leak arm identity.
 command -v codex >/dev/null 2>&1 || { echo "codex CLI not on PATH; cannot judge"; exit 1; }
-CODEX_CLI_VER=$(codex --version 2>/dev/null || echo "codex-cli unknown")
-JUDGE_MODEL=$(grep -E '^model\s*=' "${HOME}/.codex/config.toml" 2>/dev/null | head -1 | sed -E 's/.*=\s*"?([^"]+)"?.*/\1/')
+CODEX_CLI_VER=$(python3 - <<'PY'
+import subprocess
+
+try:
+    proc = subprocess.run(
+        ["codex", "--version"],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+        timeout=5,
+    )
+    out = (proc.stdout or proc.stderr).strip()
+    if proc.returncode == 0 and out:
+        print(out)
+    else:
+        print(f"codex-cli unknown (version-exit-{proc.returncode})")
+except subprocess.TimeoutExpired:
+    print("codex-cli unknown (version-timeout)")
+except FileNotFoundError:
+    print("codex-cli missing")
+except Exception as exc:
+    print(f"codex-cli unknown ({type(exc).__name__})")
+PY
+)
+JUDGE_MODEL=$({ grep -E '^model[[:space:]]*=' "${HOME}/.codex/config.toml" 2>/dev/null || true; } \
+  | head -1 \
+  | sed -E 's/.*=[[:space:]]*"?([^"]*)"?[[:space:]]*$/\1/' \
+  | xargs)
 [ -z "$JUDGE_MODEL" ] && JUDGE_MODEL="(unknown — codex config.toml not readable)"
 
 JUDGE_CWD="/tmp/judge-$RUN_ID-$FIXTURE"
@@ -237,6 +265,7 @@ fi
 
 # Extract JSON (codex wraps with banners; pick the last {...} block)
 python3 - "$JUDGE_OUT" "$RES_DIR/judge.json" "$A_ARM" "$B_ARM" "$C_ARM" "$SEED" "$CODEX_CLI_VER" "$JUDGE_MODEL" <<'PY'
+import math
 import sys, re, json, pathlib
 out = pathlib.Path(sys.argv[1]).read_text()
 target = pathlib.Path(sys.argv[2])
@@ -298,6 +327,56 @@ if axis_invalid_cells:
         f"clamped: {axis_invalid_cells}\n"
     )
 
+# Verification is the machine-readable acceptance contract. RUBRIC.md puts
+# verification behavior under Spec Compliance, but LLM judges can still grade
+# generous prose around failed commands. Cap score mechanically so an arm that
+# fails required verification cannot receive a ceiling score.
+def arm_verify_score(arm: str):
+    path = target.parent / arm / "verify.json"
+    if not path.is_file():
+        return None
+    data = json.loads(path.read_text())
+    value = data.get("verify_score")
+    return float(value) if isinstance(value, (int, float)) else None
+
+verify_caps = []
+for letter, score_key, breakdown_key in (
+    ("A", "a_score", "a_breakdown"),
+    ("B", "b_score", "b_breakdown"),
+    ("C", "c_score", "c_breakdown"),
+):
+    arm = mapping.get(letter)
+    if not arm:
+        continue
+    verify_score = arm_verify_score(arm)
+    if verify_score is None:
+        continue
+    verify_score = max(0.0, min(1.0, verify_score))
+    score_cap = math.floor(100 * verify_score)
+    spec_cap = math.floor(25 * verify_score)
+    raw_score = chosen.get(score_key)
+    raw_spec = (chosen.get(breakdown_key) or {}).get("spec")
+    row = {
+        "letter": letter,
+        "arm": arm,
+        "verify_score": verify_score,
+        "score_cap": score_cap,
+        "spec_cap": spec_cap,
+        "raw_score": raw_score,
+        "raw_spec": raw_spec,
+        "score_capped": False,
+        "spec_capped": False,
+    }
+    if isinstance(raw_score, (int, float)) and raw_score > score_cap:
+        chosen[score_key] = score_cap
+        row["score_capped"] = True
+    breakdown = chosen.get(breakdown_key)
+    if isinstance(breakdown, dict) and isinstance(raw_spec, (int, float)) and raw_spec > spec_cap:
+        breakdown["spec"] = spec_cap
+        row["spec_capped"] = True
+    verify_caps.append(row)
+chosen["_verify_score_caps"] = verify_caps
+
 # scores_by_arm: arm-name → score, computed from the blind A/B/C scores.
 # This is the canonical 3-arm-aware shape the report consumer reads. The
 # legacy variant_score / bare_score / margin fields below are derived from

package/benchmark/auto-resolve/scripts/oracle-scope-tier-a.py

@@ -154,20 +154,9 @@ def analyze(work_dir, scaffold_sha, waivers, fixture_id=None):
     findings = []
     entries = git_diff_status(scaffold_sha, work_dir)
 
-    # Structural exemption: every benchmark fixture has its own spec at
-    # docs/roadmap/phase-*/<fixture_id>.md, and auto-resolve's DOCS phase
-    # Job 1 legitimately flips its frontmatter status. That flip is a
-    # skill feature, not a scope violation — always exempt regardless of
-    # per-fixture waivers.
-    own_spec_globs = []
-    if fixture_id:
-        own_spec_globs.append(f"docs/roadmap/phase-*/{fixture_id}.md")
-
     for status, path in entries:
         if is_waived(path, waivers):
             continue
-        if is_waived(path, own_spec_globs):
-            continue
 
         # Lockfile deletion — only when file existed at scaffold.
         if status == "D" and os.path.basename(path) in LOCKFILE_NAMES:

package/benchmark/auto-resolve/scripts/oracle-scope-tier-b.py

@@ -173,22 +173,12 @@ def analyze(work_dir_str: str, scaffold_sha: str, tier_c_globs, waivers,
 
     reachable = bfs_trace(seeds, work_dir)
 
-    # Structural exemption: the fixture's own spec file at
-    # docs/roadmap/phase-*/<fixture_id>.md is always authorized — DOCS
-    # phase Job 1 flips its frontmatter status by design. Kept in sync
-    # with oracle-scope-tier-a.py.
-    own_spec_globs = []
-    if fixture_id:
-        own_spec_globs.append(f"docs/roadmap/phase-*/{fixture_id}.md")
-
     findings = []
     for path in sorted(touched):
         if match_any(path, tier_c_globs):
             continue
         if match_any(path, waivers):
             continue
-        if match_any(path, own_spec_globs):
-            continue
         if path in reachable:
             depth, via = reachable[path]
             findings.append({

package/benchmark/auto-resolve/scripts/prepare-swebench-frozen-case.py

@@ -0,0 +1,244 @@
+#!/usr/bin/env python3
+"""Prepare a SWE-bench instance for frozen VERIFY solo-vs-pair review.
+
+The script does not run models and does not evaluate SWE-bench correctness.
+It converts one official SWE-bench-style instance plus one candidate patch into
+the case layout consumed by run-frozen-verify-pair.sh.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import shlex
+import shutil
+import subprocess
+from pathlib import Path
+from typing import Any
+
+
+SAFE_ID = re.compile(r"^[A-Za-z0-9_.-]+$")
+
+
+def run(cmd: list[str], cwd: Path | None = None) -> None:
+    subprocess.run(cmd, cwd=cwd, check=True)
+
+
+def read_json(path: Path) -> dict[str, Any]:
+    with path.open(encoding="utf8") as f:
+        data = json.load(f)
+    if not isinstance(data, dict):
+        raise ValueError(f"expected JSON object: {path}")
+    return data
+
+
+def require_text(instance: dict[str, Any], key: str) -> str:
+    value = instance.get(key)
+    if not isinstance(value, str) or not value.strip():
+        raise ValueError(f"SWE-bench instance missing non-empty {key!r}")
+    return value.strip()
+
+
+def repo_cache_name(repo: str, base_commit: str) -> str:
+    safe_repo = repo.replace("/", "__")
+    return f"{safe_repo}-{base_commit[:12]}"
+
+
+def prepare_repo(instance: dict[str, Any], repo_dir: Path | None, repos_root: Path) -> Path:
+    repo = require_text(instance, "repo")
+    base_commit = require_text(instance, "base_commit")
+    repos_root.mkdir(parents=True, exist_ok=True)
+    dest = repos_root / repo_cache_name(repo, base_commit)
+
+    if repo_dir is not None:
+        if dest.exists():
+            shutil.rmtree(dest)
+        run(["git", "clone", "--quiet", "--no-hardlinks", str(repo_dir), str(dest)])
+    elif not dest.exists():
+        run(["git", "clone", "--quiet", f"https://github.com/{repo}.git", str(dest)])
+
+    run(["git", "fetch", "--quiet", "--all", "--tags"], cwd=dest)
+    run(["git", "checkout", "--quiet", base_commit], cwd=dest)
+    run(["git", "reset", "--hard", "--quiet"], cwd=dest)
+    run(["git", "clean", "-ffdqx"], cwd=dest)
+    return dest
+
+
+def write_case_files(
+    instance: dict[str, Any],
+    case_dir: Path,
+    patch_text: str,
+    timeout_seconds: int,
+) -> None:
+    instance_id = require_text(instance, "instance_id")
+    repo = require_text(instance, "repo")
+    base_commit = require_text(instance, "base_commit")
+    problem = require_text(instance, "problem_statement")
+    case_dir.mkdir(parents=True, exist_ok=True)
+
+    metadata = {
+        "id": instance_id,
+        "category": "high-risk",
+        "difficulty": instance.get("difficulty") or "swebench",
+        "timeout_seconds": timeout_seconds,
+        "required_tools": ["git", "python3"],
+        "browser": False,
+        "deps_change_expected": True,
+        "intent": f"SWE-bench issue for {repo} at {base_commit}: resolve the supplied problem statement without using the gold patch.",
+        "source": {
+            "benchmark": "SWE-bench",
+            "repo": repo,
+            "base_commit": base_commit,
+            "issue_url": instance.get("issue_url"),
+            "pr_url": instance.get("pr_url"),
+            "version": instance.get("version"),
+        },
+    }
+    (case_dir / "metadata.json").write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf8")
+
+    spec = f"""---
+id: "{instance_id}"
+title: "SWE-bench {instance_id}"
+status: planned
+complexity: high
+depends-on: []
+---
+
+# SWE-bench {instance_id}
+
+## Context
+
+Repository: `{repo}`
+Base commit: `{base_commit}`
+
+This case is imported from a SWE-bench-style instance. Treat the problem
+statement below as the visible user contract. Do not use the gold `patch` or
+`test_patch` fields as implementation guidance during model generation or
+review.
+
+## Requirements
+
+- [ ] Resolve the reported issue described in the problem statement.
+- [ ] Preserve existing behavior outside the issue's scope.
+- [ ] Keep the implementation consistent with the repository's local style and
+      dependency policy.
+- [ ] Surface failures explicitly; do not hide errors behind silent fallbacks.
+
+## Problem Statement
+
+{problem}
+
+## Constraints
+
+- Do not inspect or rely on the SWE-bench gold solution patch while producing
+  or judging a candidate patch.
+- Do not add broad rewrites, unrelated formatting churn, or new dependencies
+  unless the problem statement strictly requires them.
+- Frozen VERIFY compares reviewers on the same already-applied candidate patch;
+  it is review evidence, not a full SWE-bench solve-rate measurement.
+
+## Verification
+
+- Run the official SWE-bench evaluator separately for solve-rate evidence.
+- Use `/devlyn:resolve --verify-only` here only to compare solo vs gated pair
+  review of the frozen candidate patch against the visible problem statement.
+"""
+    (case_dir / "spec.md").write_text(spec, encoding="utf8")
+    (case_dir / "task.txt").write_text(problem + "\n", encoding="utf8")
+    (case_dir / "expected.json").write_text(
+        json.dumps(
+            {
+                "verification_commands": [],
+                "forbidden_patterns": [],
+                "required_files": [],
+                "forbidden_files": [],
+                "tier_a_waivers": [],
+                "spec_output_files": [],
+                "max_deps_added": 999,
+            },
+            indent=2,
+        )
+        + "\n",
+        encoding="utf8",
+    )
+    (case_dir / "setup.sh").write_text("#!/usr/bin/env bash\nset -euo pipefail\n", encoding="utf8")
+    (case_dir / "setup.sh").chmod(0o755)
+    notes = f"""# {instance_id} — SWE-bench Frozen VERIFY Case
+
+Source repo: `{repo}`
+Base commit: `{base_commit}`
+
+This case exists to measure whether gated pair VERIFY catches verdict-binding
+review issues that solo VERIFY misses on a fixed candidate patch. It does not
+replace official SWE-bench pass/fail evaluation.
+"""
+    (case_dir / "NOTES.md").write_text(notes, encoding="utf8")
+    (case_dir / "model.patch").write_text(patch_text, encoding="utf8")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--instance-json", required=True, type=Path)
+    parser.add_argument("--model-patch", required=True, type=Path)
+    parser.add_argument(
+        "--cases-root",
+        default=Path("benchmark/auto-resolve/external/swebench/cases"),
+        type=Path,
+    )
+    parser.add_argument(
+        "--repos-root",
+        default=Path("benchmark/auto-resolve/external/swebench/repos"),
+        type=Path,
+    )
+    parser.add_argument(
+        "--repo-dir",
+        type=Path,
+        help="Local clone/source repo to copy instead of cloning GitHub; useful for tests and cached runs.",
+    )
+    parser.add_argument("--timeout-seconds", type=int, default=2400)
+    args = parser.parse_args()
+
+    instance = read_json(args.instance_json)
+    instance_id = require_text(instance, "instance_id")
+    if not SAFE_ID.match(instance_id):
+        raise ValueError(f"unsafe instance_id for path/spec use: {instance_id!r}")
+    patch_text = args.model_patch.read_text(encoding="utf8")
+    if not patch_text.strip():
+        raise ValueError(f"model patch is empty: {args.model_patch}")
+
+    repo_path = prepare_repo(instance, args.repo_dir, args.repos_root)
+    case_dir = args.cases_root / instance_id
+    write_case_files(instance, case_dir, patch_text, args.timeout_seconds)
+
+    command = [
+        "bash",
+        "benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh",
+        "--fixture",
+        instance_id,
+        "--fixtures-root",
+        str(args.cases_root),
+        "--base-repo",
+        str(repo_path),
+        "--diff",
+        str(case_dir / "model.patch"),
+        "--pair-mode",
+        "gated",
+    ]
+    (case_dir / "run-command.txt").write_text(shlex.join(command) + "\n", encoding="utf8")
+    print(
+        json.dumps(
+            {
+                "instance_id": instance_id,
+                "case_dir": str(case_dir),
+                "repo_dir": str(repo_path),
+                "run_command": command,
+            },
+            indent=2,
+        )
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/benchmark/auto-resolve/scripts/prepare-swebench-frozen-corpus.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+"""Prepare a SWE-bench prediction JSONL as frozen VERIFY review cases."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import tempfile
+from pathlib import Path
+from typing import Any
+
+
+def read_jsonl(path: Path) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    with path.open(encoding="utf8") as f:
+        for line_no, line in enumerate(f, start=1):
+            if not line.strip():
+                continue
+            value = json.loads(line)
+            if not isinstance(value, dict):
+                raise ValueError(f"{path}:{line_no}: expected JSON object")
+            rows.append(value)
+    return rows
+
+
+def require_text(row: dict[str, Any], key: str, source: str) -> str:
+    value = row.get(key)
+    if not isinstance(value, str) or not value.strip():
+        raise ValueError(f"{source} missing non-empty {key!r}")
+    return value.strip()
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--instances-jsonl", required=True, type=Path)
+    parser.add_argument("--predictions-jsonl", required=True, type=Path)
+    parser.add_argument(
+        "--cases-root",
+        default=Path("benchmark/auto-resolve/external/swebench/cases"),
+        type=Path,
+    )
+    parser.add_argument(
+        "--repos-root",
+        default=Path("benchmark/auto-resolve/external/swebench/repos"),
+        type=Path,
+    )
+    parser.add_argument("--repo-dir", type=Path, help="Use one local repo clone for every selected instance.")
+    parser.add_argument("--instance-id", action="append", help="Prepare only these instance ids.")
+    parser.add_argument("--limit", type=int, help="Prepare at most N matched instances after filtering.")
+    parser.add_argument("--timeout-seconds", type=int, default=2400)
+    parser.add_argument("--out-manifest", type=Path)
+    args = parser.parse_args()
+
+    instances = {require_text(row, "instance_id", f"{args.instances_jsonl}"): row for row in read_jsonl(args.instances_jsonl)}
+    predictions: dict[str, dict[str, Any]] = {}
+    for row in read_jsonl(args.predictions_jsonl):
+        instance_id = require_text(row, "instance_id", f"{args.predictions_jsonl}")
+        if instance_id in predictions:
+            raise ValueError(f"duplicate prediction for {instance_id}")
+        predictions[instance_id] = row
+
+    selected_ids = args.instance_id or list(predictions)
+    script = Path(__file__).with_name("prepare-swebench-frozen-case.py")
+    prepared: list[dict[str, Any]] = []
+    with tempfile.TemporaryDirectory() as tmp:
+        tmp_dir = Path(tmp)
+        for instance_id in selected_ids:
+            if args.limit is not None and len(prepared) >= args.limit:
+                break
+            if instance_id not in instances:
+                raise ValueError(f"prediction instance not found in instances JSONL: {instance_id}")
+            prediction = predictions.get(instance_id)
+            if prediction is None:
+                raise ValueError(f"selected instance missing prediction: {instance_id}")
+            patch_value = prediction.get("model_patch")
+            if not isinstance(patch_value, str) or not patch_value.strip():
+                raise ValueError(f"prediction {instance_id} missing non-empty 'model_patch'")
+            instance_path = tmp_dir / f"{instance_id}.instance.json"
+            patch_path = tmp_dir / f"{instance_id}.patch"
+            instance_path.write_text(json.dumps(instances[instance_id], indent=2) + "\n", encoding="utf8")
+            patch_path.write_text(patch_value, encoding="utf8")
+
+            cmd = [
+                "python3",
+                str(script),
+                "--instance-json",
+                str(instance_path),
+                "--model-patch",
+                str(patch_path),
+                "--cases-root",
+                str(args.cases_root),
+                "--repos-root",
+                str(args.repos_root),
+                "--timeout-seconds",
+                str(args.timeout_seconds),
+            ]
+            if args.repo_dir is not None:
+                cmd.extend(["--repo-dir", str(args.repo_dir)])
+            completed = subprocess.run(cmd, check=True, text=True, capture_output=True)
+            prepared.append(json.loads(completed.stdout))
+
+    manifest = {
+        "instances_jsonl": str(args.instances_jsonl),
+        "predictions_jsonl": str(args.predictions_jsonl),
+        "cases_root": str(args.cases_root),
+        "repos_root": str(args.repos_root),
+        "prepared_count": len(prepared),
+        "prepared": prepared,
+    }
+    if args.out_manifest:
+        args.out_manifest.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf8")
+    print(json.dumps(manifest, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())