devlyn-cli 3.2.1 → 3.3.0

package/benchmark/auto-resolve/scripts/prepare-swebench-solver-worktree.py

@@ -1,210 +0,0 @@
-#!/usr/bin/env python3
-"""Prepare a SWE-bench instance worktree for producing a candidate patch."""
-
-from __future__ import annotations
-
-import argparse
-import json
-import re
-import shutil
-import subprocess
-from pathlib import Path
-from typing import Any
-
-from pair_evidence_contract import reject_json_constant
-
-
-SAFE_ID = re.compile(r"^[A-Za-z0-9_.-]+$")
-SAFE_REPO = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")
-SAFE_COMMIT = re.compile(r"^[0-9a-fA-F]{7,40}$")
-
-
-def run(cmd: list[str], cwd: Path | None = None) -> None:
-    subprocess.run(cmd, cwd=cwd, check=True)
-
-
-def read_instances(path: Path) -> list[dict[str, Any]]:
-    rows: list[dict[str, Any]] = []
-    with path.open(encoding="utf8") as f:
-        for line_no, line in enumerate(f, start=1):
-            if not line.strip():
-                continue
-            value = json.loads(line, parse_constant=reject_json_constant)
-            if not isinstance(value, dict):
-                raise ValueError(f"{path}:{line_no}: expected JSON object")
-            rows.append(value)
-    return rows
-
-
-def require_text(instance: dict[str, Any], key: str) -> str:
-    value = instance.get(key)
-    if not isinstance(value, str) or not value.strip():
-        raise ValueError(f"SWE-bench instance missing non-empty {key!r}")
-    return value.strip()
-
-
-def require_safe_repo(instance: dict[str, Any]) -> str:
-    repo = require_text(instance, "repo")
-    if not SAFE_REPO.match(repo):
-        raise ValueError(f"unsafe SWE-bench repo: {repo!r}")
-    return repo
-
-
-def require_safe_base_commit(instance: dict[str, Any]) -> str:
-    base_commit = require_text(instance, "base_commit")
-    if not SAFE_COMMIT.match(base_commit):
-        raise ValueError(f"unsafe SWE-bench base_commit: {base_commit!r}")
-    return base_commit
-
-
-def pick_instance(path: Path, instance_id: str) -> dict[str, Any]:
-    matches = [row for row in read_instances(path) if row.get("instance_id") == instance_id]
-    if len(matches) != 1:
-        raise ValueError(f"expected exactly one {instance_id!r} row in {path}, found {len(matches)}")
-    return matches[0]
-
-
-def repo_cache_name(repo: str, base_commit: str) -> str:
-    return f"{repo.replace('/', '__')}-{base_commit[:12]}"
-
-
-def prepare_repo(instance: dict[str, Any], repos_root: Path) -> Path:
-    repo = require_safe_repo(instance)
-    base_commit = require_safe_base_commit(instance)
-    repos_root.mkdir(parents=True, exist_ok=True)
-    dest = repos_root / repo_cache_name(repo, base_commit)
-
-    if not dest.exists():
-        run(["git", "clone", "--quiet", f"https://github.com/{repo}.git", str(dest)])
-
-    run(["git", "fetch", "--quiet", "--all", "--tags"], cwd=dest)
-    run(["git", "checkout", "--quiet", base_commit], cwd=dest)
-    run(["git", "reset", "--hard", "--quiet"], cwd=dest)
-    run(["git", "clean", "-ffdqx"], cwd=dest)
-    return dest
-
-
-def copy_worktree(repo_path: Path, worktree: Path) -> None:
-    if worktree.exists():
-        shutil.rmtree(worktree)
-    run(["git", "clone", "--quiet", "--no-hardlinks", str(repo_path), str(worktree)])
-    run(["git", "checkout", "--quiet", "HEAD"], cwd=worktree)
-    run(["git", "reset", "--hard", "--quiet"], cwd=worktree)
-    run(["git", "clean", "-ffdqx"], cwd=worktree)
-
-
-def write_spec(instance: dict[str, Any], worktree: Path) -> Path:
-    instance_id = require_text(instance, "instance_id")
-    repo = require_safe_repo(instance)
-    base_commit = require_safe_base_commit(instance)
-    problem = require_text(instance, "problem_statement")
-    spec_path = worktree / "docs" / "roadmap" / "phase-1" / f"{instance_id}.md"
-    spec_path.parent.mkdir(parents=True, exist_ok=True)
-    spec_path.write_text(
-        f"""---
-id: "{instance_id}"
-title: "SWE-bench {instance_id}"
-status: planned
-complexity: high
-depends-on: []
----
-
-# SWE-bench {instance_id}
-
-Repository: `{repo}`
-Base commit: `{base_commit}`
-
-## Requirements
-
-- [ ] Resolve the issue described in the problem statement.
-- [ ] Preserve existing behavior outside the issue's scope.
-- [ ] Keep the implementation consistent with the repository's local style and
-      dependency policy.
-- [ ] Add focused regression coverage when practical.
-
-## Problem Statement
-
-{problem}
-
-## Constraints
-
-- Do not inspect or rely on the SWE-bench gold `patch` or `test_patch` fields.
-- Do not add broad rewrites, unrelated formatting churn, or new dependencies
-  unless the visible problem statement strictly requires them.
-
-## Verification
-
-- Run the most focused practical verification for the changed behavior.
-""",
-        encoding="utf8",
-    )
-    return spec_path
-
-
-def copy_devlyn_context(worktree: Path) -> None:
-    skills_src = Path("config/skills")
-    if skills_src.exists():
-        skills_dst = worktree / ".claude" / "skills"
-        if skills_dst.exists():
-            shutil.rmtree(skills_dst)
-        shutil.copytree(skills_src, skills_dst)
-    claude_src = Path("CLAUDE.md")
-    if claude_src.exists():
-        shutil.copy2(claude_src, worktree / "CLAUDE.md")
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--instances-jsonl", required=True, type=Path)
-    parser.add_argument("--instance-id", required=True)
-    parser.add_argument(
-        "--repos-root",
-        default=Path("benchmark/auto-resolve/external/swebench/repos-solver"),
-        type=Path,
-    )
-    parser.add_argument(
-        "--worktrees-root",
-        default=Path("benchmark/auto-resolve/external/swebench/worktrees"),
-        type=Path,
-    )
-    parser.add_argument("--copy-devlyn-context", action="store_true")
-    args = parser.parse_args()
-
-    instance = pick_instance(args.instances_jsonl, args.instance_id)
-    instance_id = require_text(instance, "instance_id")
-    if not SAFE_ID.match(instance_id):
-        raise ValueError(f"unsafe instance_id for path/spec use: {instance_id!r}")
-
-    repo_path = prepare_repo(instance, args.repos_root)
-    worktree = args.worktrees_root / instance_id
-    args.worktrees_root.mkdir(parents=True, exist_ok=True)
-    copy_worktree(repo_path, worktree)
-    spec_path = write_spec(instance, worktree)
-    if args.copy_devlyn_context:
-        copy_devlyn_context(worktree)
-
-    prompt = (
-        f"You are solving SWE-bench instance {instance_id} in this checked-out repository at "
-        "the base commit. Do not inspect any gold SWE-bench patch or test_patch. Read the "
-        f"local code and the spec at {spec_path.relative_to(worktree)}. Make the smallest "
-        "correct source/test change for the visible issue. Run a focused verification "
-        "command. At the end, report changed files, verification command, and verdict."
-    )
-    (worktree / "solve-prompt.txt").write_text(prompt + "\n", encoding="utf8")
-    print(
-        json.dumps(
-            {
-                "instance_id": instance_id,
-                "repo_dir": str(repo_path),
-                "worktree": str(worktree),
-                "spec_path": str(spec_path),
-                "prompt_file": str(worktree / "solve-prompt.txt"),
-            },
-            indent=2,
-        )
-    )
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

package/benchmark/auto-resolve/scripts/recent-benchmark-summary.py

@@ -1,232 +0,0 @@
-#!/usr/bin/env python3
-"""Print a compact, wrap-safe benchmark snapshot from local artifacts."""
-from __future__ import annotations
-
-import argparse
-import importlib.util
-import json
-import pathlib
-import sys
-import textwrap
-from typing import Any
-
-SCRIPT_DIR = pathlib.Path(__file__).resolve().parent
-FRONTIER_PATH = SCRIPT_DIR / "pair-candidate-frontier.py"
-
-
-def load_frontier_module() -> Any:
-    spec = importlib.util.spec_from_file_location("pair_candidate_frontier", FRONTIER_PATH)
-    if spec is None or spec.loader is None:
-        raise RuntimeError(f"cannot load frontier module: {FRONTIER_PATH}")
-    module = importlib.util.module_from_spec(spec)
-    spec.loader.exec_module(module)
-    return module
-
-
-FRONTIER = load_frontier_module()
-
-
-def best_rows(report: dict[str, Any]) -> list[dict[str, Any]]:
-    rows: list[dict[str, Any]] = []
-    for row in report.get("rows", []):
-        if row.get("status") != "pair_evidence_passed":
-            continue
-        best = FRONTIER.best_pair_evidence(row.get("passing_pair_evidence", []))
-        if best is None:
-            continue
-        rows.append({"fixture": row["fixture"], **best})
-    return rows
-
-
-def display_fixture(fixture: str) -> str:
-    short, _, rest = fixture.partition("-")
-    return f"{short} {rest.replace('-', ' ')}" if rest else fixture
-
-
-def fmt_margin(value: Any) -> str:
-    return f"{value:+d}" if isinstance(value, int) and not isinstance(value, bool) else "n/a"
-
-
-def fmt_decimal_margin(value: Any) -> str:
-    return f"{value:+.2f}" if isinstance(value, (int, float)) and not isinstance(value, bool) else "n/a"
-
-
-def fmt_wall(value: Any) -> str:
-    return f"{value:.2f}x" if isinstance(value, (int, float)) and not isinstance(value, bool) else "n/a"
-
-
-def fmt_score(value: Any) -> str:
-    return str(value) if isinstance(value, int) and not isinstance(value, bool) else "n/a"
-
-
-def wrap_item(prefix: str, text: str, *, width: int) -> list[str]:
-    return textwrap.wrap(
-        text,
-        width=width,
-        initial_indent=prefix,
-        subsequent_indent=" " * len(prefix),
-        break_long_words=False,
-        break_on_hyphens=False,
-    ) or [prefix.rstrip()]
-
-
-def render_text(report: dict[str, Any], *, width: int) -> str:
-    rows = best_rows(report)
-    lines = [
-        "Recent Benchmark Snapshot",
-        "=========================",
-        "",
-        "Status",
-        f"  Verdict: {report.get('verdict', 'n/a')}",
-        f"  Active fixtures: {report.get('fixtures_total', 'n/a')}",
-        f"  Rejected controls: {report.get('rejected_total', 'n/a')}",
-        f"  Pair evidence rows: {report.get('pair_evidence_total', 'n/a')}",
-        f"  Unmeasured candidates: {report.get('unmeasured_candidate_total', 'n/a')}",
-        "",
-        "Pair Lift",
-        f"  Average margin: {fmt_decimal_margin(report.get('pair_margin_avg'))}",
-        f"  Minimum margin: {fmt_margin(report.get('pair_margin_min'))}",
-        f"  Average wall ratio: {fmt_wall(report.get('pair_solo_wall_ratio_avg'))}",
-        f"  Maximum wall ratio: {fmt_wall(report.get('pair_solo_wall_ratio_max'))}",
-        f"  Gate: margin >= {fmt_margin(report.get('min_pair_margin'))}; wall <= {fmt_wall(report.get('max_pair_solo_wall_ratio'))}",
-        "",
-        "Pair Evidence",
-    ]
-    if not rows:
-        lines.append("  No passing pair evidence rows found.")
-        return "\n".join(lines) + "\n"
-
-    for item in rows:
-        lines.append(f"  {display_fixture(item['fixture'])}")
-        lines.append(
-            "    scores: bare {bare} | solo_claude {solo} | pair {pair}".format(
-                bare=fmt_score(item.get("bare_score")),
-                solo=fmt_score(item.get("solo_score")),
-                pair=fmt_score(item.get("pair_score")),
-            )
-        )
-        lines.append(
-            "    lift: {margin} | wall {wall} | arm {arm}".format(
-                margin=fmt_margin(item.get("pair_margin")),
-                wall=fmt_wall(item.get("pair_solo_wall_ratio")),
-                arm=item.get("pair_arm") or "n/a",
-            )
-        )
-        lines.extend(wrap_item("    run: ", str(item.get("run_id") or "n/a"), width=width))
-        triggers = ", ".join(item.get("pair_trigger_reasons") or [])
-        lines.extend(wrap_item("    triggers: ", triggers or "n/a", width=width))
-    return "\n".join(lines) + "\n"
-
-
-def render_markdown(report: dict[str, Any], *, width: int) -> str:
-    rows = best_rows(report)
-    lines = [
-        "# Recent Benchmark Snapshot",
-        "",
-        "## Status",
-        "",
-        f"- Verdict: **{report.get('verdict', 'n/a')}**",
-        f"- Active fixtures: {report.get('fixtures_total', 'n/a')}",
-        f"- Rejected controls: {report.get('rejected_total', 'n/a')}",
-        f"- Pair evidence rows: {report.get('pair_evidence_total', 'n/a')}",
-        f"- Unmeasured candidates: {report.get('unmeasured_candidate_total', 'n/a')}",
-        "",
-        "## Pair Lift",
-        "",
-        f"- Average margin: **{fmt_decimal_margin(report.get('pair_margin_avg'))}**",
-        f"- Minimum margin: **{fmt_margin(report.get('pair_margin_min'))}**",
-        f"- Average wall ratio: {fmt_wall(report.get('pair_solo_wall_ratio_avg'))}",
-        f"- Maximum wall ratio: {fmt_wall(report.get('pair_solo_wall_ratio_max'))}",
-        f"- Gate: margin >= {fmt_margin(report.get('min_pair_margin'))}; wall <= {fmt_wall(report.get('max_pair_solo_wall_ratio'))}",
-        "",
-        "## Pair Evidence",
-        "",
-    ]
-    if not rows:
-        lines.append("No passing pair evidence rows found.")
-        return "\n".join(lines) + "\n"
-
-    for item in rows:
-        lines.extend(
-            [
-                f"### {display_fixture(item['fixture'])}",
-                "",
-                f"- Scores: bare {fmt_score(item.get('bare_score'))}, solo_claude {fmt_score(item.get('solo_score'))}, pair {fmt_score(item.get('pair_score'))}.",
-                f"- Lift: {fmt_margin(item.get('pair_margin'))}; wall {fmt_wall(item.get('pair_solo_wall_ratio'))}; arm `{item.get('pair_arm') or 'n/a'}`.",
-                f"- Run: `{item.get('run_id') or 'n/a'}`.",
-            ]
-        )
-        triggers = ", ".join(item.get("pair_trigger_reasons") or [])
-        wrapped = wrap_item("- Triggers: ", triggers or "n/a", width=width)
-        lines.extend(wrapped)
-        lines.append("")
-    return "\n".join(lines).rstrip() + "\n"
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--fixtures-root",
-        type=pathlib.Path,
-        default=pathlib.Path("benchmark/auto-resolve/fixtures"),
-    )
-    parser.add_argument(
-        "--registry",
-        type=pathlib.Path,
-        default=SCRIPT_DIR / "pair-rejected-fixtures.sh",
-    )
-    parser.add_argument(
-        "--results-root",
-        type=pathlib.Path,
-        default=pathlib.Path("benchmark/auto-resolve/results"),
-    )
-    parser.add_argument("--out-json", type=pathlib.Path)
-    parser.add_argument("--out-md", type=pathlib.Path)
-    parser.add_argument(
-        "--max-width",
-        type=int,
-        default=92,
-        help="target maximum line width for text and markdown output",
-    )
-    parser.add_argument(
-        "--min-pair-margin",
-        type=int,
-        default=5,
-        help="minimum pair-over-solo margin required to count passing pair evidence",
-    )
-    parser.add_argument(
-        "--max-pair-solo-wall-ratio",
-        type=float,
-        default=3.0,
-        help="maximum pair/solo wall-time ratio allowed to count passing pair evidence",
-    )
-    args = parser.parse_args()
-    if args.max_width < 60:
-        print("error: --max-width must be >= 60", file=sys.stderr)
-        return 2
-
-    try:
-        report = FRONTIER.build_report(
-            fixtures_root=args.fixtures_root,
-            registry=args.registry,
-            results_root=args.results_root,
-            min_pair_margin=args.min_pair_margin,
-            max_pair_solo_wall_ratio=args.max_pair_solo_wall_ratio,
-        )
-    except ValueError as exc:
-        print(f"error: {exc}", file=sys.stderr)
-        return 2
-
-    if args.out_json:
-        args.out_json.parent.mkdir(parents=True, exist_ok=True)
-        args.out_json.write_text(json.dumps(report, indent=2) + "\n", encoding="utf8")
-    if args.out_md:
-        args.out_md.parent.mkdir(parents=True, exist_ok=True)
-        args.out_md.write_text(render_markdown(report, width=args.max_width), encoding="utf8")
-
-    print(render_text(report, width=args.max_width), end="")
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())