@elizaos/sweagent-root 2.0.0-alpha

package/python/sweagent/utils/config.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any
+
+from dotenv import load_dotenv
+
+from sweagent import REPO_ROOT
+from sweagent.utils.log import get_logger
+
+logger = get_logger("swea-config", emoji="🔧")
+
+
+def _convert_path_relative_to_repo_root(path: Path | str, root: Path | None = None) -> Path | str:
+    original_type = type(path)
+    path = Path(path).resolve()
+    root = Path(root or os.getenv("SWE_AGENT_CONFIG_ROOT", REPO_ROOT))
+    relative_path = path.relative_to(root) if root in path.parents else path
+    return relative_path if original_type is Path else str(relative_path)
+
+
+def _could_be_a_path(v: Any) -> bool:
+    try:
+        return Path(v).exists()
+    except Exception:
+        return False
+
+
+def _strip_abspath_from_dict(value: dict | list | str, root: Path | None = None) -> dict | list | str:
+    root = Path(root or os.getenv("SWE_AGENT_CONFIG_ROOT", REPO_ROOT))
+    if isinstance(value, dict):
+        return {k: _strip_abspath_from_dict(v, root) for k, v in value.items()}
+    elif isinstance(value, list):
+        return [_strip_abspath_from_dict(v, root) for v in value]
+    elif isinstance(value, str) and _could_be_a_path(value):
+        return _convert_path_relative_to_repo_root(value, root)
+    else:
+        return value
+
+
+def _convert_path_to_abspath(path: Path | str) -> Path:
+    """If path is not absolute, convert it to an absolute path
+    using the SWE_AGENT_CONFIG_ROOT environment variable (if set) or
+    REPO_ROOT as base.
+    """
+    path = Path(path)
+    root = Path(os.getenv("SWE_AGENT_CONFIG_ROOT", REPO_ROOT))
+    assert root.is_dir()
+    if not path.is_absolute():
+        path = root / path
+    assert path.is_absolute()
+    return path.resolve()
+
+
+def _convert_paths_to_abspath(paths: list[Path] | list[str]) -> list[Path]:
+    return [_convert_path_to_abspath(p) for p in paths]
+
+
+def load_environment_variables(path: Path | None = None):
+    """Load environment variables from a .env file.
+    If path is not provided, we first look for a .env file in the current working
+    directory and then in the repository root.
+    """
+    if path is None:
+        cwd_path = Path.cwd() / ".env"
+        repo_path = REPO_ROOT / ".env"
+        if cwd_path.exists():
+            path = cwd_path
+        elif repo_path.exists():
+            path = REPO_ROOT / ".env"
+        else:
+            logger.debug("No .env file found")
+            return
+    if not path.is_file():
+        msg = f"No .env file found at {path}"
+        raise FileNotFoundError(msg)
+    anything_loaded = load_dotenv(dotenv_path=path)
+    if anything_loaded:
+        logger.info(f"Loaded environment variables from {path}")

package/python/sweagent/utils/files.py

@@ -0,0 +1,27 @@
+import json
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+def load_file(path: Path | str | None) -> Any:
+    """Load files based on their extension."""
+    if path is None:
+        return None
+    if isinstance(path, str):
+        path = Path(path)
+    if not path.exists():
+        raise FileNotFoundError(path)
+    if path.is_dir():
+        from datasets import load_from_disk
+
+        return load_from_disk(path)
+    if path.suffix in [".json", ".traj"]:
+        return json.loads(path.read_text())
+    if path.suffix == ".jsonl":
+        return [json.loads(line) for line in path.read_text().splitlines() if line.strip()]
+    if path.suffix == ".yaml":
+        return yaml.safe_load(path.read_text())
+    msg = f"Unsupported file extension: {path.suffix}"
+    raise NotImplementedError(msg)

package/python/sweagent/utils/github.py

@@ -0,0 +1,118 @@
+import re
+
+from ghapi.all import GhApi
+
+GITHUB_ISSUE_URL_PATTERN = re.compile(r"github\.com\/(.*?)\/(.*?)\/issues\/(\d+)")
+
+
+class InvalidGithubURL(Exception):
+    """Raised when a github URL is invalid"""
+
+
+GITHUB_REPO_URL_PATTERN = re.compile(r".*[/@]?github\.com\/([^/]+)\/([^/]+)")
+
+
+def _is_github_repo_url(data_path: str) -> bool:
+    """Check if data_path is an URL pointing to a github repository.
+    Paths to issues or PRs will also match this pattern.
+    """
+    return GITHUB_REPO_URL_PATTERN.search(data_path) is not None
+
+
+def _is_github_issue_url(data_path: str) -> bool:
+    """Check if data_path is an URL pointing to a github issue"""
+    return GITHUB_ISSUE_URL_PATTERN.search(data_path) is not None
+
+
+def _get_commit(api: GhApi, owner: str, repo: str, ref: str | None = None):
+    """Get commit object from github api
+
+    Args:
+        api (GhApi):
+        owner (str): Repo owner, e.g., "SWE-agent"
+        repo (str): Repo, e.g., "SWE-agent"
+        ref (str, optional): Branch, tag or commit hash
+
+    Returns:
+        _type_: _description_
+    """
+    if ref:
+        return api.repos.get_commit(owner, repo, ref)  # type: ignore
+    return api.repos.list_commits(owner, repo)[0]  # type: ignore
+
+
+def _parse_gh_issue_url(issue_url: str) -> tuple[str, str, str]:
+    """
+    Returns:
+        owner: Repo owner
+        repo: Repo name
+        issue number: Issue number as str
+
+    Raises:
+        InvalidGithubURL: If the URL is not a valid github issue URL
+    """
+    match = GITHUB_ISSUE_URL_PATTERN.search(issue_url)
+    if not match:
+        msg = f"Invalid GitHub issue URL: {issue_url}"
+        raise InvalidGithubURL(msg)
+    res = match.groups()
+    assert len(res) == 3
+    return tuple(res)  # type: ignore
+
+
+def _parse_gh_repo_url(repo_url: str) -> tuple[str, str]:
+    """
+    Returns:
+        owner: Repo owner/org
+        repo: Repo name
+
+    Raises:
+        InvalidGithubURL: If the URL is not a valid github repo URL
+    """
+    match = GITHUB_REPO_URL_PATTERN.search(repo_url)
+    if not match:
+        msg = f"Invalid GitHub issue URL: {repo_url}"
+        raise InvalidGithubURL(msg)
+    res = match.groups()
+    assert len(res) == 2
+    return tuple(res)  # type: ignore
+
+
+def _get_gh_issue_data(issue_url: str, *, token: str = ""):
+    """Returns github issue data in the form of a dictionary.
+    See https://docs.github.com/en/rest/issues/issues?apiVersion=2022-11-28#get-an-issue
+    for return format
+    """
+    owner, repo, issue_number = _parse_gh_issue_url(issue_url)
+    api = GhApi(token=token)
+    return api.issues.get(owner, repo, issue_number)  # type: ignore
+
+
+def _get_problem_statement_from_github_issue(
+    owner: str, repo: str, issue_number: str, *, token: str | None = ""
+) -> str:
+    """Return problem statement from github issue"""
+    api = GhApi(token=token)
+    issue = api.issues.get(owner, repo, issue_number)  # type: ignore
+    title = issue.title if issue.title else ""
+    body = issue.body if issue.body else ""
+    return f"{title}\n{body}\n"
+
+
+def _get_associated_commit_urls(org: str, repo: str, issue_number: str, *, token: str = "") -> list[str]:
+    """Return the URLs of commits that would close an issue."""
+    api = GhApi(token=token)
+    # Strangely the "pull_request" field of api.issues.get is often not set
+    # so we have to go through the events to check if there's a commit
+    events = api.issues.list_events(org, repo, issue_number)  # type: ignore
+    commit_urls = []
+    for event in events:
+        if event.event != "referenced":
+            continue
+        if not event.commit_id:
+            continue
+        commit = api.repos.get_commit(org, repo, event.commit_id)  # type: ignore
+        message = commit.commit.message
+        if f"fixes #{issue_number}" in message.lower() or f"closes #{issue_number}" in message.lower():
+            commit_urls.append(commit.html_url)
+    return commit_urls

package/python/sweagent/utils/jinja_warnings.py

@@ -0,0 +1,14 @@
+from sweagent.utils.log import get_logger
+
+
+def _warn_probably_wrong_jinja_syntax(template: str | None) -> None:
+    """Warn if the template uses {var} instead of {{var}}."""
+    if template is None:
+        return
+    if "{" not in template:
+        return
+    for s in ["{%", "{ %", "{{"]:
+        if s in template:
+            return
+    logger = get_logger("swea-config", emoji="🔧")
+    logger.warning("Probably wrong Jinja syntax in template: %s. Make sure to use {{var}} instead of {var}.", template)

package/python/sweagent/utils/log.py

@@ -0,0 +1,175 @@
+from __future__ import annotations
+
+import logging
+import os
+import threading
+import uuid
+from collections.abc import Callable
+from pathlib import Path, PurePath
+
+from rich.logging import RichHandler
+from rich.text import Text
+
+_SET_UP_LOGGERS: set[str] = set()
+_ADDITIONAL_HANDLERS: dict[str, logging.Handler] = {}
+_LOG_LOCK = threading.Lock()
+
+logging.TRACE = 5  # type: ignore
+logging.addLevelName(logging.TRACE, "TRACE")  # type: ignore
+
+
+def _interpret_level(level: int | str | None, *, default=logging.DEBUG) -> int:
+    if not level:
+        return default
+    if isinstance(level, int):
+        return level
+    if level.isnumeric():
+        return int(level)
+    return getattr(logging, level.upper())
+
+
+_STREAM_LEVEL = _interpret_level(os.environ.get("SWE_AGENT_LOG_STREAM_LEVEL"))
+_INCLUDE_LOGGER_NAME_IN_STREAM_HANDLER = False
+
+_THREAD_NAME_TO_LOG_SUFFIX: dict[str, str] = {}
+"""Mapping from thread name to suffix to add to the logger name."""
+
+
+def register_thread_name(name: str) -> None:
+    """Register a suffix to add to the logger name for the current thread."""
+    thread_name = threading.current_thread().name
+    _THREAD_NAME_TO_LOG_SUFFIX[thread_name] = name
+
+
+class _RichHandlerWithEmoji(RichHandler):
+    def __init__(self, emoji: str, *args, **kwargs):
+        """Subclass of RichHandler that adds an emoji to the log message."""
+        super().__init__(*args, **kwargs)
+        if not emoji.endswith(" "):
+            emoji += " "
+        self.emoji = emoji
+
+    def get_level_text(self, record: logging.LogRecord) -> Text:
+        level_name = record.levelname.replace("WARNING", "WARN")
+        return Text.styled((self.emoji + level_name).ljust(10), f"logging.level.{level_name.lower()}")
+
+
+def get_logger(name: str, *, emoji: str = "") -> logging.Logger:
+    """Get logger. Use this instead of `logging.getLogger` to ensure
+    that the logger is set up with the correct handlers.
+    """
+    thread_name = threading.current_thread().name
+    if thread_name != "MainThread":
+        name = name + "-" + _THREAD_NAME_TO_LOG_SUFFIX.get(thread_name, thread_name)
+    logger = logging.getLogger(name)
+    if logger.hasHandlers():
+        # Already set up
+        return logger
+    handler = _RichHandlerWithEmoji(
+        emoji=emoji,
+        show_time=bool(os.environ.get("SWE_AGENT_LOG_TIME", False)),
+        show_path=False,
+    )
+    handler.setLevel(_STREAM_LEVEL)
+    # Set to lowest level and only use stream handlers to adjust levels
+    logger.setLevel(logging.TRACE)  # type: ignore
+    logger.addHandler(handler)
+    logger.propagate = False
+    _SET_UP_LOGGERS.add(name)
+    with _LOG_LOCK:
+        for handler in _ADDITIONAL_HANDLERS.values():
+            my_filter = getattr(handler, "my_filter", None)
+            if my_filter is None:
+                logger.addHandler(handler)
+            elif isinstance(my_filter, str) and my_filter in name:
+                logger.addHandler(handler)
+            elif callable(my_filter) and my_filter(name):
+                logger.addHandler(handler)
+    if _INCLUDE_LOGGER_NAME_IN_STREAM_HANDLER:
+        _add_logger_name_to_stream_handler(logger)
+    return logger
+
+
+def add_file_handler(
+    path: PurePath | str,
+    *,
+    filter: str | Callable[[str], bool] | None = None,
+    level: int | str = logging.TRACE,  # type: ignore[attr-defined]
+    id_: str = "",
+) -> str:
+    """Adds a file handler to all loggers that we have set up
+    and all future loggers that will be set up with `get_logger`.
+
+    Args:
+        filter: If str: Check that the logger name contains the filter string.
+            If callable: Check that the logger name satisfies the condition returned by the callable.
+        level: The level of the handler.
+        id_: The id of the handler. If not provided, a random id will be generated.
+
+    Returns:
+        The id of the handler. This can be used to remove the handler later.
+    """
+    Path(path).parent.mkdir(parents=True, exist_ok=True)
+    handler = logging.FileHandler(path, encoding="utf-8")
+    formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(name)s - %(message)s")
+    handler.setFormatter(formatter)
+    handler.setLevel(_interpret_level(level))
+    with _LOG_LOCK:
+        # Lock because other thread might be modifying the _SET_UP_LOGGERS set
+        for name in _SET_UP_LOGGERS:
+            if filter is not None:
+                if isinstance(filter, str) and filter not in name:
+                    continue
+                if callable(filter) and not filter(name):
+                    continue
+            logger = logging.getLogger(name)
+            logger.addHandler(handler)
+    handler.my_filter = filter  # type: ignore
+    if not id_:
+        id_ = str(uuid.uuid4())
+    _ADDITIONAL_HANDLERS[id_] = handler
+    return id_
+
+
+def remove_file_handler(id_: str) -> None:
+    """Remove a file handler by its id."""
+    handler = _ADDITIONAL_HANDLERS.pop(id_)
+    with _LOG_LOCK:
+        # Lock because other thread might be modifying the _SET_UP_LOGGERS set
+        for log_name in _SET_UP_LOGGERS:
+            logger = logging.getLogger(log_name)
+            logger.removeHandler(handler)
+
+
+def _add_logger_name_to_stream_handler(logger: logging.Logger) -> None:
+    for handler in logger.handlers:
+        if isinstance(handler, _RichHandlerWithEmoji):
+            formatter = logging.Formatter("[%(name)s] %(message)s")
+            handler.setFormatter(formatter)
+
+
+def add_logger_names_to_stream_handlers() -> None:
+    """Add the logger name to the stream handler for all loggers that we have set up."""
+    global _INCLUDE_LOGGER_NAME_IN_STREAM_HANDLER
+    _INCLUDE_LOGGER_NAME_IN_STREAM_HANDLER = True
+    with _LOG_LOCK:
+        for logger in _SET_UP_LOGGERS:
+            _add_logger_name_to_stream_handler(logging.getLogger(logger))
+
+
+def set_stream_handler_levels(level: int) -> None:
+    """Set the default stream level and adjust the levels of all stream handlers
+    to be at most the given level.
+
+    Note: Can only be used to lower the level, not raise it.
+    """
+    global _STREAM_LEVEL
+    _STREAM_LEVEL = level
+    with _LOG_LOCK:
+        for name in _SET_UP_LOGGERS:
+            logger = logging.getLogger(name)
+            for handler in logger.handlers:
+                if isinstance(handler, _RichHandlerWithEmoji):
+                    current_level = handler.level
+                    if current_level < level:
+                        handler.setLevel(level)

package/python/sweagent/utils/patch_formatter.py

@@ -0,0 +1,152 @@
+from collections.abc import Callable
+
+from unidiff import PatchSet
+
+
+class PatchFormatter:
+    def __init__(
+        self,
+        patch: str,
+        read_method: Callable[[str], str],
+    ):
+        """Given the final patch and access to the container that contains the repository,
+        extract relevant lines from the modified file.
+
+        Args:
+            patch: The patch as a string.
+            read_method: Callable with path to file (relative to repository root) as argument
+                that returns the file content as a string.
+        """
+        self._patch = PatchSet(patch)
+        self._patched_files: dict[str, str] = {}
+        self._original_files: dict[str, str] = {}
+        self._patch_applied = True
+        self._read_file = read_method
+        self._read_files(original=False)
+
+    @staticmethod
+    def _merge_intervals(starts: list[int], stops: list[int]) -> tuple[list[int], list[int]]:
+        """Given two lists of integers, starts and stops, merges all overlapping intervals.
+
+        For example `starts=[1, 5, 18]`, `stops=[10, 13, 20]`
+        should return `starts=[1, 18]`, `stops=[13, 20]`
+        """
+        if not starts:
+            assert not stops
+            return [], []
+
+        intervals = sorted(zip(starts, stops))
+        merged = []
+        for start, stop in intervals:
+            if not merged or merged[-1][1] < start:
+                # No overlap
+                merged.append([start, stop])
+            else:
+                # Overlap
+                merged[-1][1] = max(merged[-1][1], stop)
+        # Unzip again
+        merged_starts, merged_stops = zip(*merged)
+        return list(merged_starts), list(merged_stops)
+
+    def format_file(self, text: str, starts: list[int], stops: list[int], *, linenos: bool = True) -> str:
+        """Reads file and returns string representation of the relevant lines.
+
+        Args:
+            path: The path to the file within the repo location
+            starts: The starting line numbers of the relevant lines. The first line is line 1.
+            stops: The stopping line numbers of the relevant lines. The stop is not inclusive.
+                The first line is line 1.
+            linenos: Whether to include line numbers
+        """
+        if not starts:
+            assert not stops
+            return ""
+
+        assert len(starts) == len(stops)
+        assert all(start >= 1 for start in starts)
+        assert all(start < stop for start, stop in zip(starts, stops))
+        starts, stops = self._merge_intervals(starts, stops)
+        assert all(hunk1_start < hunk2_start for hunk1_start, hunk2_start in zip(starts, starts[1:]))
+        out: list[str] = []
+        if starts[0] > 1:
+            # Count from 1
+            out.append(f"[{starts[0] - 1} lines above omitted]")
+        last_stop: int | None = None
+        lines = text.splitlines()
+        for start, stop in zip(starts, stops):
+            assert start >= 1
+            if last_stop is not None:
+                n_omitted = start - last_stop
+                # Check that we have non-overlapping hunks
+                assert n_omitted >= 0
+                if n_omitted:
+                    out.append(f"\n[{n_omitted} lines omitted]\n")
+            # Count from 1
+            these_lines = lines[start - 1 : stop - 1]
+            if linenos:
+                out.append("\n".join([f"{i:6d}: {l}" for i, l in enumerate(these_lines, start=start)]))
+            else:
+                out.append("\n".join(these_lines))
+            last_stop = stop
+        if last_stop < len(lines):
+            # Stop is not inclusive
+            omitted = len(lines) - last_stop
+            assert omitted > 0
+            out.append(f"[{omitted} lines below omitted]")
+        return "\n".join(out)
+
+    def _get_hunk_lines(self, original: bool, *, context_length: int) -> dict[str, tuple[list[int], list[int]]]:
+        """Get the starts and stops for all files in the patch.
+
+        Args:
+            original: Whether to read the original file or the patched file
+            context_length: The number of lines to include above and below the hunk
+
+        Returns:
+            A dictionary with the file path as key and a tuple of lists of starts and stops as value.
+        """
+        out: dict[str, tuple[list[int], list[int]]] = {}
+        for patch in self._patch:
+            if not patch.is_modified_file:
+                continue
+            starts: list[int] = []
+            stops: list[int] = []
+            for hunk in patch:
+                if original:
+                    # 1 is the lowest line number
+                    start = max(1, hunk.source_start - context_length)
+                    stop = hunk.source_start + hunk.source_length + context_length
+                else:
+                    start = max(1, hunk.target_start - context_length)
+                    stop = hunk.target_start + hunk.target_length + context_length
+                starts.append(start)
+                stops.append(stop)
+            out[patch.path] = (starts, stops)
+        return out
+
+    def _read_files(self, original: bool) -> None:
+        for patch in self._patch:
+            path = patch.path
+            if not patch.is_modified_file:
+                continue
+            if original:
+                msg = "Original file reading not implemented"
+                raise NotImplementedError(msg)
+            else:
+                assert self._patch_applied
+                self._patched_files[path] = self._read_file(path)
+
+    @staticmethod
+    def concat_files_strings(files: dict[str, str]) -> str:
+        """Concatenate multiple `read_files` outputs into a single string."""
+        out = []
+        for path, content in files.items():
+            out.append(f"[File: {path}]\n{content}")
+        return "\n\n".join(out)
+
+    def get_files_str(self, *, original: bool, context_length: int | None = 50, linenos: bool = True) -> str:
+        hunk_lines = self._get_hunk_lines(original=original, context_length=context_length)
+        sources = self._original_files if original else self._patched_files
+        return self.concat_files_strings(
+            {path: self.format_file(text, *hunk_lines[path], linenos=linenos) for path, text in sources.items()}
+        )

package/python/sweagent/utils/serialization.py

@@ -0,0 +1,45 @@
+import io
+from copy import deepcopy
+from typing import Any
+
+from ruamel.yaml import YAML
+from ruamel.yaml.scalarstring import LiteralScalarString as LSS
+
+
+def _convert_to_yaml_literal_string(d: Any) -> Any:
+    """Convert any multi-line strings in nested data object to LiteralScalarString.
+    This will then use the `|-` syntax of yaml.
+    """
+    d = deepcopy(d)
+    if isinstance(d, dict):
+        for key, value in d.items():
+            d[key] = _convert_to_yaml_literal_string(value)
+    elif isinstance(d, list):
+        for i, item in enumerate(d):
+            d[i] = _convert_to_yaml_literal_string(item)
+    elif isinstance(d, str) and "\n" in d:
+        d = LSS(d.replace("\r\n", "\n").replace("\r", "\n"))
+    return d
+
+
+def _yaml_serialization_with_linebreaks(data: Any) -> str:
+    data = _convert_to_yaml_literal_string(data)
+    yaml = YAML()
+    yaml.indent(mapping=2, sequence=4, offset=2)
+    yaml.width = float("inf")
+    yaml.default_flow_style = False
+    buffer = io.StringIO()
+    yaml.dump(data, buffer)
+    return buffer.getvalue()
+
+
+def merge_nested_dicts(d1: dict, d2: dict) -> dict:
+    """Merge two nested dictionaries, updating d1 in place.
+    If a key exists in both dictionaries, the value from d2 will be used.
+    """
+    for key, value in d2.items():
+        if isinstance(value, dict):
+            d1[key] = merge_nested_dicts(d1.get(key, {}), value)
+        else:
+            d1[key] = value
+    return d1