@elizaos/sweagent-root 2.0.0-alpha

package/python/sweagent/run/hooks/apply_patch.py

@@ -0,0 +1,106 @@
+import subprocess
+from pathlib import Path
+
+import rich
+import rich.markdown
+import rich.panel
+
+from sweagent.agent.problem_statement import ProblemStatementConfig
+from sweagent.environment.repo import LocalRepoConfig
+from sweagent.environment.swe_env import SWEEnv
+from sweagent.run.common import _is_promising_patch
+from sweagent.run.hooks.abstract import RunHook
+from sweagent.types import AgentRunResult
+from sweagent.utils.log import get_logger
+
+
+class SaveApplyPatchHook(RunHook):
+    """This hook saves patches to a separate directory and optionally applies them to a local repository."""
+
+    def __init__(self, apply_patch_locally: bool = False, show_success_message: bool = True):
+        self.logger = get_logger("swea-save_apply_patch", emoji="⚡️")
+        self._apply_patch_locally = apply_patch_locally
+        self._show_success_message = show_success_message
+
+    def on_init(self, *, run):
+        self._output_dir = Path(run.output_dir)
+
+    def on_instance_start(self, *, index: int, env: SWEEnv, problem_statement: ProblemStatementConfig):
+        self._env = env
+        self._problem_statement = problem_statement
+
+    def on_instance_completed(self, *, result: AgentRunResult):
+        instance_id = self._problem_statement.id
+        patch_path = self._save_patch(instance_id, result.info)
+        if patch_path:
+            if not self._apply_patch_locally:
+                return
+            if not _is_promising_patch(result.info):
+                return
+            if self._env.repo is None:
+                return
+            if not isinstance(self._env.repo, LocalRepoConfig):
+                return
+            local_dir = Path(self._env.repo.path)
+            self._apply_patch(patch_path, local_dir)
+
+    @staticmethod
+    def _print_patch_message(patch_output_file: Path):
+        console = rich.console.Console()
+        msg = [
+            "SWE-agent has produced a patch that it believes will solve the issue you submitted!",
+            "Use the code snippet below to inspect or apply it!",
+        ]
+        panel = rich.panel.Panel.fit(
+            "\n".join(msg),
+            title="🎉 Submission successful 🎉",
+        )
+        console.print(panel)
+        content = [
+            "```bash",
+            "# The patch has been saved to your local filesystem at:",
+            f"PATCH_FILE_PATH='{patch_output_file.resolve()}'",
+            "# Inspect it:",
+            'cat "${PATCH_FILE_PATH}"',
+            "# Apply it to a local repository:",
+            "cd <your local repo root>",
+            'git apply "${PATCH_FILE_PATH}"',
+            "```",
+        ]
+        console.print(rich.markdown.Markdown("\n".join(content)))
+
+    def _save_patch(self, instance_id: str, info) -> Path | None:
+        """Create patch files that can be applied with `git am`.
+
+        Returns:
+            The path to the patch file, if it was saved. Otherwise, returns None.
+        """
+        patch_output_dir = self._output_dir / instance_id
+        patch_output_dir.mkdir(exist_ok=True, parents=True)
+        patch_output_file = patch_output_dir / f"{instance_id}.patch"
+        if info.get("submission") is None:
+            self.logger.info("No patch to save.")
+            return None
+        model_patch = info["submission"]
+        patch_output_file.write_text(model_patch)
+        if _is_promising_patch(info):
+            # Only print big congratulations if we actually believe
+            # the patch will solve the issue
+            if self._show_success_message:
+                self._print_patch_message(patch_output_file)
+        return patch_output_file
+
+    def _apply_patch(self, patch_file: Path, local_dir: Path) -> None:
+        """Apply a patch to a local directory."""
+
+        assert local_dir.is_dir()
+        assert patch_file.exists()
+        # The resolve() is important, because we're gonna run the cmd
+        # somewhere else
+        cmd = ["git", "apply", str(patch_file.resolve())]
+        try:
+            subprocess.run(cmd, cwd=local_dir, check=True)
+        except subprocess.CalledProcessError as e:
+            self.logger.error(f"Failed to apply patch {patch_file} to {local_dir}: {e}")
+            return
+        self.logger.info(f"Applied patch {patch_file} to {local_dir}")

package/python/sweagent/run/hooks/open_pr.py

@@ -0,0 +1,244 @@
+import os
+import random
+import shlex
+
+from ghapi.all import GhApi
+from pydantic import BaseModel
+
+from sweagent.environment.swe_env import SWEEnv
+from sweagent.run.hooks.abstract import RunHook
+from sweagent.types import AgentRunResult
+from sweagent.utils.github import (
+    InvalidGithubURL,
+    _get_associated_commit_urls,
+    _get_gh_issue_data,
+    _parse_gh_issue_url,
+)
+from sweagent.utils.log import get_logger
+
+# NOTE
+# THE IMPLEMENTATION DETAILS HERE WILL CHANGE SOON!
+
+
+# fixme: Bring back the ability to open the PR to a fork
+def open_pr(*, logger, token, env: SWEEnv, github_url, trajectory, _dry_run: bool = False) -> None:
+    """Create PR to repository
+
+    Args:
+        trajectory: Trajectory of actions taken by the agent
+        _dry_run: Whether to actually push anything or just simulate it
+    """
+
+    issue_url = github_url
+    logger.info("Opening PR")
+    try:
+        issue = _get_gh_issue_data(issue_url, token=token)
+    except InvalidGithubURL as e:
+        msg = "Data path must be a github issue URL if open_pr is set to True."
+        raise ValueError(msg) from e
+    branch_name = f"swe-agent-fix-#{issue.number}-" + str(random.random())[2:10]
+    env.communicate(
+        input="git config user.email 'noemail@swe-agent.com' && git config user.name 'SWE-agent'",
+        error_msg="Failed to set git user",
+        timeout=10,
+        check="raise",
+    )
+    env.communicate(input="rm -f model.patch", error_msg="Failed to remove model patch", timeout=10, check="raise")
+    env.communicate(
+        input=f"git checkout -b {branch_name}", error_msg="Failed to switch to new branch", timeout=10, check="raise"
+    )
+    env.communicate(input="git add .", error_msg="Failed to add commits", timeout=10, check="raise")
+    dry_run_flag = "--allow-empty" if _dry_run else ""
+    commit_msg = [
+        shlex.quote(f"Fix: {issue.title}"),
+        shlex.quote(f"Closes #{issue.number}"),
+    ]
+    out = env.communicate(
+        input=f"git commit -m {commit_msg[0]} -m  {commit_msg[1]} {dry_run_flag}",
+        error_msg="Failed to commit changes",
+        timeout=10,
+        check="raise",
+    )
+    logger.debug(f"Committed changes: {out}")
+
+    owner, repo, _ = _parse_gh_issue_url(issue_url)
+    # fixme: bring this back
+    # If `--repo_path` was specified with a different github URL, then the record will contain
+    # the forking user
+    forker = owner
+    head = branch_name
+    remote = "origin"
+    if forker != owner:
+        head = f"{forker}:{branch_name}"
+        token_prefix = ""
+        if token:
+            token_prefix = f"{token}@"
+        fork_url = f"https://{token_prefix}github.com/{forker}/{repo}.git"
+        logger.debug(f"Using fork: {fork_url}")
+        env.communicate(
+            input=f"git remote add fork {fork_url}",
+            error_msg="Failed to create new git remote",
+            timeout=10,
+        )
+        remote = "fork"
+    dry_run_prefix = "echo " if _dry_run else ""
+    out = env.communicate(
+        input=f"{dry_run_prefix} git push {remote} {branch_name}",
+        error_msg=(
+            "Failed to push branch to remote. Please check your token and permissions. "
+            "You might want to push to a fork with the push_gh_repo_url option."
+        ),
+        timeout=10,
+    )
+    logger.debug(f"Pushed commit to {remote=} {branch_name=}: {out}")
+    body = (
+        f"This is a PR opened by AI tool [SWE Agent](https://github.com/SWE-agent/SWE-agent/) "
+        f"to close [#{issue.number}]({issue_url}) ({issue.title}).\n\nCloses #{issue.number}."
+    )
+    body += "\n\n" + format_trajectory_markdown(trajectory, char_limit=60_000)
+    api = GhApi(token=token)
+    default_branch = api.repos.get(owner, repo).default_branch
+    if not _dry_run:
+        args = dict(
+            owner=owner,
+            repo=repo,
+            title=f"SWE-agent[bot] PR to fix: {issue.title}",
+            head=head,
+            base=default_branch,
+            body=body,
+            draft=True,
+        )
+        logger.debug(f"Creating PR with args: {args}")
+        pr_info = api.pulls.create(**args)  # type: ignore
+        logger.info(
+            f"🎉 PR created as a draft at {pr_info.html_url}. Please review it carefully, push "
+            "any required changes onto the branch and then click "
+            "'Ready for Review' to bring it to the attention of the maintainers.",
+        )
+
+
+class OpenPRConfig(BaseModel):
+    # Option to be used with open_pr: Skip action if there are already commits claiming
+    # to fix the issue. Please only set this to False if you are sure the commits are
+    # not fixes or if this is your own repository!
+    skip_if_commits_reference_issue: bool = True
+
+
+class OpenPRHook(RunHook):
+    """This hook opens a PR if the issue is solved and the user has enabled the option."""
+
+    def __init__(self, config: OpenPRConfig):
+        self.logger = get_logger("swea-open_pr", emoji="⚡️")
+        self._config = config
+
+    def on_init(self, *, run):
+        self._env = run.env
+        self._token: str = os.getenv("GITHUB_TOKEN", "")
+        self._problem_statement = run.problem_statement
+
+    def on_instance_completed(self, result: AgentRunResult):
+        if self.should_open_pr(result):
+            open_pr(
+                logger=self.logger,
+                token=self._token,
+                env=self._env,
+                github_url=self._problem_statement.github_url,
+                trajectory=result.trajectory,
+            )
+
+    def should_open_pr(self, result: AgentRunResult) -> bool:
+        """Does opening a PR make sense?"""
+        if not result.info.get("submission"):
+            self.logger.info("Not opening PR because no submission was made.")
+            return False
+        if result.info.get("exit_status") != "submitted":
+            self.logger.info(
+                "Not opening PR because exit status was %s and not submitted.", result.info.get("exit_status")
+            )
+            return False
+        try:
+            issue = _get_gh_issue_data(self._problem_statement.github_url, token=self._token)
+        except InvalidGithubURL:
+            self.logger.info("Currently only GitHub is supported to open PRs to. Skipping PR creation.")
+            return False
+        if issue.state != "open":
+            self.logger.info(f"Issue is not open (state={issue.state}. Skipping PR creation.")
+            return False
+        if issue.assignee:
+            self.logger.info("Issue is already assigned. Skipping PR creation. Be nice :)")
+            return False
+        if issue.locked:
+            self.logger.info("Issue is locked. Skipping PR creation.")
+            return False
+        org, repo, issue_number = _parse_gh_issue_url(self._problem_statement.github_url)
+        associated_commits = _get_associated_commit_urls(org, repo, issue_number, token=self._token)
+        if associated_commits:
+            commit_url_strs = ", ".join(associated_commits)
+            if self._config.skip_if_commits_reference_issue:
+                self.logger.info(f"Issue already has associated commits (see {commit_url_strs}). Skipping PR creation.")
+                return False
+            else:
+                self.logger.warning(
+                    "Proceeding with PR creation even though there are already commits "
+                    f"({commit_url_strs}) associated with the issue. Please only do this for your own repositories "
+                    "or after verifying that the existing commits do not fix the issue.",
+                )
+        return True
+
+
+def _remove_triple_backticks(text: str) -> str:
+    return "\n".join(line.removeprefix("```") for line in text.splitlines())
+
+
+def format_trajectory_markdown(trajectory: list[dict[str, str]], char_limit: int | None = None):
+    """Format a trajectory as a markdown string for use in gh PR description.
+
+    Args:
+        char_limit: If not None, truncate the trajectory to this many characters.
+    """
+    prefix = [
+        "<details>",
+        "<summary>Thought process ('trajectory') of SWE-agent (click to expand)</summary>",
+        "",
+        "",
+    ]
+    prefix_text = "\n".join(prefix)
+    suffix = [
+        "",
+        "</details>",
+    ]
+    suffix_text = "\n".join(suffix)
+
+    steps = []
+    current_length = len(prefix_text) + len(suffix_text)
+
+    for i, step in enumerate(trajectory):
+        step_strs = [
+            f"**🧑‍🚒 Response ({i})**: ",
+            f"{step['response'].strip()}",
+            f"**👀‍ Observation ({i})**:",
+            "```",
+            f"{_remove_triple_backticks(step['observation']).strip()}",
+            "```",
+        ]
+        step_text = "\n".join(step_strs)
+
+        # Calculate separator length (only needed for steps after the first one)
+        separator_length = 0
+        if steps:
+            separator_length = len("\n\n---\n\n")
+
+        # Check if adding this step would exceed the character limit
+        if char_limit is not None and current_length + separator_length + len(step_text) > char_limit:
+            if i > 0:
+                steps.append("\n\n... (truncated due to length limit)")
+            break
+
+        if steps:
+            steps.append("\n\n---\n\n")
+            current_length += separator_length
+
+        steps.append(step_text)
+        current_length += len(step_text)
+
+    return prefix_text + "".join(steps) + suffix_text

package/python/sweagent/run/hooks/swe_bench_evaluate.py

@@ -0,0 +1,113 @@
+"""SweBench evaluation hook.
+
+Will be automatically added to `run_batch` if `SWEBenchInstances.evaluate` is set to true
+"""
+
+import subprocess
+import sys
+from datetime import datetime
+from pathlib import Path
+from threading import Lock
+from time import time
+
+from sweagent.run.hooks.abstract import RunHook
+from sweagent.run.merge_predictions import merge_predictions
+from sweagent.types import AgentRunResult
+from sweagent.utils.log import get_logger
+
+
+class SweBenchEvaluate(RunHook):
+    _SUBSET_MAP = {"lite": "swe-bench_lite", "verified": "swe-bench_verified", "multimodal": "swe-bench_multimodal"}
+
+    def __init__(self, output_dir: Path, subset: str, split: str, continuous_submission_every: int = 0) -> None:
+        super().__init__()
+        self.output_dir = output_dir
+        self.subset = subset
+        self.split = split
+        self.continuous_submission_every = continuous_submission_every
+        self.logger = get_logger("SB-evaluate", emoji="😬")
+        self.merge_lock = Lock()
+        self.last_evaluation_time = time()
+        self.evaluation_interval = continuous_submission_every
+        self._running_calls = []
+        # We need to add a suffix to the run_id to avoid collisions when you reuse the name of your run
+        self._time_suffix = datetime.now().strftime("%Y%m%d%H%M%S%f")
+
+    @property
+    def run_id(self) -> str:
+        return f"{self.output_dir.name}_{self._time_suffix}"
+
+    def _get_sb_call(self, preds_path: Path, submit_only: bool = False) -> list[str]:
+        args = [
+            "sb-cli",
+            "submit",
+            self._SUBSET_MAP[self.subset],
+            self.split,
+            "--predictions_path",
+            str(preds_path),
+            "--run_id",
+            self.run_id,
+            "--output_dir",
+            str(self.output_dir / "sb-cli-reports"),
+        ]
+        if submit_only:
+            args.extend(["--wait_for_evaluation", "0", "--gen_report", "0", "--verify_submission", "0"])
+        return args
+
+    def check_running_calls(self) -> None:
+        """Warn if one of the running calls failed."""
+        for call in self._running_calls:
+            if call.poll() is not None:
+                if call.returncode != 0:
+                    self.logger.error("Failed to submit results to SweBench eval: %s", call.stderr.read())
+                self._running_calls.remove(call)
+
+    def on_instance_completed(self, *, result: AgentRunResult):
+        if self.evaluation_interval == 0:
+            return
+
+        current_time = time()
+        if current_time - self.last_evaluation_time < self.evaluation_interval:
+            return
+
+        with self.merge_lock:
+            merge_predictions([self.output_dir], self.output_dir / "tmppreds.json")
+            self.last_evaluation_time = current_time
+
+        self._running_calls.append(
+            subprocess.Popen(
+                self._get_sb_call(preds_path=self.output_dir / "tmppreds.json", submit_only=True),
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            )
+        )
+
+    def move_sb_cli_report(self) -> None:
+        """Move report from `sb-cli-reports` to `results.json`."""
+        output_dir = self.output_dir / "sb-cli-reports"
+        if not output_dir.exists():
+            self.logger.warning("No SweBench report found at %s", output_dir)
+            return
+        (self.output_dir / "results.json").unlink(missing_ok=True)
+        reports = list(output_dir.glob("*.json"))
+        if len(reports) != 1:
+            self.logger.warning("Expected 1 SweBench report at %s, found %d. Cannot rename.", output_dir, len(reports))
+            return
+        reports[0].rename(self.output_dir / "results.json")
+
+    def on_end(self) -> None:
+        self.logger.info("Submitting results to SWE-Bench")
+        try:
+            subprocess.run(
+                self._get_sb_call(preds_path=self.output_dir / "preds.json"),
+                check=True,
+                stdout=sys.stdout,
+                stderr=sys.stderr,
+            )
+        except subprocess.CalledProcessError as e:
+            self.logger.error("Failed to submit results to SweBench eval: %s", e)
+        else:
+            # remove temporary predictions if they exist
+            if (self.output_dir / "tmppreds.json").exists():
+                (self.output_dir / "tmppreds.json").unlink()
+            self.move_sb_cli_report()