@elizaos/sweagent-root 2.0.0-alpha

package/python/tests/test_run.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+import subprocess
+
+
+def test_run_cli_no_arg_error():
+    args = [
+        "sweagent",
+    ]
+    output = subprocess.run(args, check=False, capture_output=True)
+    print(output.stdout.decode())
+    print(output.stderr.decode())
+    assert output.returncode in [1, 2]
+    assert "run-batch" in output.stdout.decode()
+    assert "run-replay" in output.stdout.decode()
+    assert "run" in output.stdout.decode()
+
+
+def test_run_cli_main_help():
+    args = [
+        "sweagent",
+        "--help",
+    ]
+    output = subprocess.run(args, check=True, capture_output=True)
+    assert "run-batch" in output.stdout.decode()
+    assert "run-replay" in output.stdout.decode()
+    assert "run" in output.stdout.decode()
+
+
+def test_run_cli_subcommand_help():
+    args = [
+        "sweagent",
+        "run",
+        "--help",
+    ]
+    output = subprocess.run(args, check=True, capture_output=True)
+    assert "--config" in output.stdout.decode()

package/python/tests/test_run_batch.py

@@ -0,0 +1,110 @@
+from pathlib import Path
+
+import pytest
+
+from sweagent.run.run import main
+
+
+@pytest.mark.slow
+def test_expert_instances(test_data_sources_path: Path, tmp_path: Path):
+    ds_path = test_data_sources_path / "expert_instances.yaml"
+    assert ds_path.exists()
+    cmd = [
+        "run-batch",
+        "--agent.model.name",
+        "instant_empty_submit",
+        "--instances.type",
+        "expert_file",
+        "--instances.path",
+        str(ds_path),
+        "--output_dir",
+        str(tmp_path),
+        "--raise_exceptions",
+        "True",
+    ]
+    main(cmd)
+    for _id in ["simple_test_problem", "simple_test_problem_2"]:
+        assert (tmp_path / f"{_id}" / f"{_id}.traj").exists(), list(tmp_path.iterdir())
+
+
+@pytest.mark.slow
+def test_simple_instances(test_data_sources_path: Path, tmp_path: Path):
+    ds_path = test_data_sources_path / "simple_instances.yaml"
+    assert ds_path.exists()
+    cmd = [
+        "run-batch",
+        "--agent.model.name",
+        "instant_empty_submit",
+        "--instances.path",
+        str(ds_path),
+        "--output_dir",
+        str(tmp_path),
+        "--raise_exceptions",
+        "True",
+    ]
+    main(cmd)
+    assert (tmp_path / "simple_test_problem" / "simple_test_problem.traj").exists(), list(tmp_path.iterdir())
+
+
+def test_empty_instances_simple(test_data_sources_path: Path, tmp_path: Path):
+    ds_path = test_data_sources_path / "simple_instances.yaml"
+    assert ds_path.exists()
+    cmd = [
+        "run-batch",
+        "--agent.model.name",
+        "instant_empty_submit",
+        "--instances.path",
+        str(ds_path),
+        "--output_dir",
+        str(tmp_path),
+        "--raise_exceptions",
+        "True",
+        "--instances.filter",
+        "doesnotmatch",
+    ]
+    with pytest.raises(ValueError, match="No instances to run"):
+        main(cmd)
+
+
+def test_empty_instances_expert(test_data_sources_path: Path, tmp_path: Path):
+    ds_path = test_data_sources_path / "expert_instances.yaml"
+    assert ds_path.exists()
+    cmd = [
+        "run-batch",
+        "--agent.model.name",
+        "instant_empty_submit",
+        "--instances.path",
+        str(ds_path),
+        "--instances.type",
+        "expert_file",
+        "--output_dir",
+        str(tmp_path),
+        "--raise_exceptions",
+        "True",
+        "--instances.filter",
+        "doesnotmatch",
+    ]
+    with pytest.raises(ValueError, match="No instances to run"):
+        main(cmd)
+
+
+# This doesn't work because we need to retrieve environment variables from the environment
+# in order to format our templates.
+# def test_run_batch_swe_bench_instances(tmp_path: Path):
+#     cmd = [
+#         "run-batch",
+#         "--agent.model.name",
+#         "instant_empty_submit",
+#         "--instances.subset",
+#         "lite",
+#         "--instances.split",
+#         "test",
+#         "--instances.slice",
+#         "0:1",
+#         "--output_dir",
+#         str(tmp_path),
+#         "--raise_exceptions",
+#         "--instances.deployment.type",
+#         "dummy",
+#     ]
+#     main(cmd)

package/python/tests/test_run_hooks.py

@@ -0,0 +1,114 @@
+import os
+
+import pytest
+
+from sweagent.agent.problem_statement import GithubIssue
+from sweagent.run.hooks.open_pr import OpenPRConfig, OpenPRHook
+from sweagent.types import AgentRunResult
+
+
+def _fake_issue_from_url(url: str):
+    # Minimal object shape used by OpenPRHook.should_open_pr
+    issue_number = url.rstrip("/").split("/")[-1]
+
+    class Issue:
+        def __init__(self, state: str, assignee, locked: bool):
+            self.state = state
+            self.assignee = assignee
+            self.locked = locked
+
+    if issue_number == "16":
+        return Issue(state="closed", assignee=None, locked=False)
+    if issue_number == "17":
+        return Issue(state="open", assignee="someone", locked=False)
+    if issue_number == "18":
+        return Issue(state="open", assignee=None, locked=True)
+    if issue_number == "19":
+        return Issue(state="open", assignee=None, locked=False)
+    return Issue(state="open", assignee=None, locked=False)
+
+
+def _fake_associated_commit_urls(_org: str, _repo: str, issue_number: str, *, token: str = ""):
+    if issue_number == "19":
+        return ["https://github.com/swe-agent/test-repo/commit/abc123"]
+    return []
+
+
+@pytest.fixture
+def open_pr_hook_init_for_sop(monkeypatch: pytest.MonkeyPatch):
+    # Patch network calls in OpenPRHook
+    import sweagent.run.hooks.open_pr as open_pr
+
+    def fake_get_issue(url: str, token: str = ""):
+        if "github.com" not in url or "/issues/" not in url:
+            msg = f"Invalid GitHub issue URL: {url}"
+            raise open_pr.InvalidGithubURL(msg)
+        return _fake_issue_from_url(url)
+
+    monkeypatch.setattr(open_pr, "_get_gh_issue_data", fake_get_issue)
+    monkeypatch.setattr(open_pr, "_get_associated_commit_urls", _fake_associated_commit_urls)
+
+    hook = OpenPRHook(config=OpenPRConfig(skip_if_commits_reference_issue=True))
+    hook._token = os.environ.get("GITHUB_TOKEN", "")
+    hook._problem_statement = GithubIssue(github_url="https://github.com/swe-agent/test-repo/issues/1")
+    return hook
+
+
+@pytest.fixture
+def agent_run_result():
+    return AgentRunResult(
+        info={
+            "submission": "asdf",
+            "exit_status": "submitted",
+        },
+        trajectory=[],
+    )
+
+
+def test_should_open_pr_fail_submission(open_pr_hook_init_for_sop, agent_run_result):
+    hook = open_pr_hook_init_for_sop
+    agent_run_result.info["submission"] = None
+    assert not hook.should_open_pr(agent_run_result)
+
+
+def test_should_open_pr_fail_exit(open_pr_hook_init_for_sop, agent_run_result):
+    hook = open_pr_hook_init_for_sop
+    agent_run_result.info["exit_status"] = "fail"
+    assert not hook.should_open_pr(agent_run_result)
+
+
+def test_should_open_pr_fail_invalid_url(open_pr_hook_init_for_sop, agent_run_result):
+    hook = open_pr_hook_init_for_sop
+    hook._problem_statement = type("PS", (), {"github_url": "asdf"})()
+    assert not hook.should_open_pr(agent_run_result)
+
+
+def test_should_open_pr_fail_closed(open_pr_hook_init_for_sop, agent_run_result):
+    hook = open_pr_hook_init_for_sop
+    hook._problem_statement = GithubIssue(github_url="https://github.com/swe-agent/test-repo/issues/16")
+    assert not hook.should_open_pr(agent_run_result)
+
+
+def test_should_open_pr_fail_assigned(open_pr_hook_init_for_sop, agent_run_result):
+    hook = open_pr_hook_init_for_sop
+    hook._problem_statement = GithubIssue(github_url="https://github.com/swe-agent/test-repo/issues/17")
+    assert not hook.should_open_pr(agent_run_result)
+
+
+def test_should_open_pr_fail_locked(open_pr_hook_init_for_sop, agent_run_result):
+    hook = open_pr_hook_init_for_sop
+    hook._problem_statement = GithubIssue(github_url="https://github.com/swe-agent/test-repo/issues/18")
+    assert not hook.should_open_pr(agent_run_result)
+
+
+def test_should_open_pr_fail_has_pr(open_pr_hook_init_for_sop, agent_run_result):
+    hook = open_pr_hook_init_for_sop
+    hook._problem_statement = GithubIssue(github_url="https://github.com/swe-agent/test-repo/issues/19")
+    assert not hook.should_open_pr(agent_run_result)
+
+
+def test_should_open_pr_success_has_pr_override(open_pr_hook_init_for_sop, agent_run_result):
+    hook = open_pr_hook_init_for_sop
+    hook._problem_statement = GithubIssue(github_url="https://github.com/swe-agent/test-repo/issues/19")
+    hook._config.skip_if_commits_reference_issue = False
+    assert hook.should_open_pr(agent_run_result)

package/python/tests/test_run_replay.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+import subprocess
+
+import pytest
+from swerex.deployment.config import DockerDeploymentConfig
+
+from sweagent.run.run_replay import RunReplay, RunReplayConfig
+
+
+@pytest.fixture
+def rr_config(swe_agent_test_repo_traj, tmp_path, swe_agent_test_repo_clone):
+    return RunReplayConfig(
+        traj_path=swe_agent_test_repo_traj,
+        deployment=DockerDeploymentConfig(image="python:3.11"),
+        output_dir=tmp_path,
+    )
+
+
+def test_replay(rr_config):
+    rr = RunReplay.from_config(rr_config, _catch_errors=False, _require_zero_exit_code=True)
+    rr.main()
+
+
+def test_run_cli_help():
+    args = [
+        "sweagent",
+        "run-replay",
+        "--help",
+    ]
+    output = subprocess.run(args, capture_output=True)
+    assert output.returncode == 0
+    assert "Replay a trajectory file" in output.stdout.decode()

package/python/tests/test_run_single.py

@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from sweagent import CONFIG_DIR, TOOLS_DIR
+from sweagent.agent.agents import DefaultAgentConfig
+from sweagent.agent.models import InstantEmptySubmitModelConfig
+from sweagent.environment.swe_env import EnvironmentConfig
+from sweagent.run.common import BasicCLI
+from sweagent.run.hooks.abstract import RunHook
+from sweagent.run.run_single import RunSingle, RunSingleConfig
+from sweagent.tools.bundle import Bundle
+
+
+class RaisesExceptionHook(RunHook):
+    def on_instance_start(self, *args, **kwargs):
+        msg = "test exception"
+        raise ValueError(msg)
+
+
+@pytest.mark.slow
+def test_run_single_raises_exception():
+    rsc = RunSingleConfig(agent=DefaultAgentConfig(model=InstantEmptySubmitModelConfig()))
+    rs = RunSingle.from_config(rsc)
+    rs.add_hook(RaisesExceptionHook())
+    with pytest.raises(ValueError, match="test exception"):
+        rs.run()
+
+
+@pytest.fixture
+def agent_config_with_commands():
+    ac = DefaultAgentConfig(model=InstantEmptySubmitModelConfig())
+    ac.tools.bundles = [
+        Bundle(path=TOOLS_DIR / "registry"),
+        Bundle(path=TOOLS_DIR / "windowed"),
+        Bundle(path=TOOLS_DIR / "submit"),
+    ]
+    ac.tools.env_variables = {"WINDOW": 100}
+    assert (TOOLS_DIR / "submit").exists()
+    # Make sure dependent properties are set
+    ac.tools.model_post_init(None)
+    return ac
+
+
+@pytest.mark.slow
+def test_hidden_tools(tmpdir):
+    ac = DefaultAgentConfig(model=InstantEmptySubmitModelConfig())
+    ac.tools.env_variables = {"WINDOW": 100}
+    ac.tools.bundles = [
+        Bundle(path=TOOLS_DIR / "registry"),
+        Bundle(path=TOOLS_DIR / "windowed", hidden_tools=["scroll_up"]),
+        Bundle(path=TOOLS_DIR / "submit"),
+    ]
+    ac.model.name = "instant_empty_submit"
+    rsc = RunSingleConfig(
+        env=EnvironmentConfig(),
+        agent=ac,
+        output_dir=Path(tmpdir),
+    )
+    rs = RunSingle.from_config(rsc)
+    rs.run()
+
+
+@pytest.mark.slow
+def test_run_ies(tmpdir, agent_config_with_commands):
+    # ies = instant empty submit
+    ac = agent_config_with_commands
+    ac.model.name = "instant_empty_submit"
+    rsc = RunSingleConfig(
+        env=EnvironmentConfig(),
+        agent=ac,
+        output_dir=Path(tmpdir),
+    )
+    rs = RunSingle.from_config(rsc)
+    rs.agent.tools.mock_state = {"open_file": "asdf123", "working_dir": "/root"}
+    rs.run()
+
+
+@pytest.mark.slow
+@pytest.mark.parametrize("repo", ["local", "github"])
+@pytest.mark.parametrize("problem_statement_source", ["github", "local", "text"])
+def test_run_ies_repo_ps_matrix(
+    tmpdir,
+    swe_agent_test_repo_clone,
+    repo,
+    problem_statement_source,
+):
+    output_formats = ["traj", "pred", "patch"]
+    for fmt in output_formats:
+        assert not list(Path(tmpdir).glob(f"*.{fmt}"))
+    if problem_statement_source == "github":
+        ps_args = ["--problem_statement.github_url", "https://github.com/swe-agent/test-repo/issues/1"]
+    elif problem_statement_source == "local":
+        ps_args = ["--problem_statement.path", str(swe_agent_test_repo_clone / "problem_statements" / "1.md")]
+    elif problem_statement_source == "text":
+        ps_args = ["--problem_statement.text='this is a test'"]
+    else:
+        raise ValueError(problem_statement_source)
+    if repo == "local":
+        repo_args = ["--env.repo.path", str(swe_agent_test_repo_clone)]
+    elif repo == "github":
+        repo_args = ["--env.repo.github_url", "https://github.com/swe-agent/test-repo"]
+    else:
+        raise ValueError(repo)
+    args = [
+        "--agent.model.name=instant_empty_submit",
+        "--output_dir",
+        str(tmpdir),
+        *ps_args,
+        *repo_args,
+        "--config",
+        str(CONFIG_DIR / "default_backticks.yaml"),
+    ]
+    print(args)
+    rs_config = BasicCLI(RunSingleConfig).get_config(args)
+    print(rs_config)
+    rs = RunSingle.from_config(rs_config)  # type: ignore
+    with tmpdir.as_cwd():
+        # Test that we can run run.py also independently from repo dir
+        rs.run()
+    for fmt in output_formats:
+        print(fmt, list(Path(tmpdir).iterdir()))
+        assert len(list(Path(tmpdir).rglob(f"*.{fmt}"))) == 1

package/python/tests/test_tools_command_parsing.py

@@ -0,0 +1,193 @@
+import pytest
+
+from sweagent.tools.commands import Argument, Command
+
+
+def test_command_parsing_formats():
+    """Test various signature formats and default parsing."""
+    # Default format (no signature)
+    command = Command(
+        name="test_cmd",
+        docstring="A test command",
+        arguments=[
+            Argument(name="arg1", type="string", description="First argument", required=True),
+            Argument(name="arg2", type="integer", description="Second argument", required=False),
+        ],
+    )
+    assert command.invoke_format == "test_cmd {arg1} {arg2} "
+
+    # Angle brackets
+    command = Command(
+        name="goto",
+        signature="goto <line_number>",
+        docstring="moves the window to show line_number",
+        arguments=[Argument(name="line_number", type="integer", description="line number", required=True)],
+    )
+    assert command.invoke_format == "goto {line_number}"
+
+    # Optional brackets (stripped in invoke_format)
+    command = Command(
+        name="open",
+        signature='open "<path>" [<line_number>]',
+        docstring="opens file at path",
+        arguments=[
+            Argument(name="path", type="string", description="file path", required=True),
+            Argument(name="line_number", type="integer", description="line number", required=False),
+        ],
+    )
+    assert command.invoke_format == 'open "{path}" {line_number}'
+
+    # Flag-style arguments
+    command = Command(
+        name="grep",
+        signature="grep --pattern <pattern> --file <file>",
+        docstring="search for pattern in file",
+        arguments=[
+            Argument(name="pattern", type="string", description="search pattern", required=True),
+            Argument(name="file", type="string", description="file to search", required=True),
+        ],
+    )
+    assert command.invoke_format == "grep --pattern {pattern} --file {file}"
+
+    # No arguments
+    command = Command(name="scroll_up", signature="scroll_up", docstring="scrolls up", arguments=[])
+    assert command.invoke_format == "scroll_up"
+
+
+def test_argument_validation():
+    """Test argument validation rules."""
+    # Required arguments must come before optional ones
+    with pytest.raises(ValueError, match="Required argument.*cannot come after optional arguments"):
+        Command(
+            name="bad_order",
+            docstring="bad argument order",
+            arguments=[
+                Argument(name="optional", type="string", description="optional", required=False),
+                Argument(name="required", type="string", description="required", required=True),
+            ],
+        )
+
+    # Duplicate argument names
+    with pytest.raises(ValueError, match="Duplicate argument names"):
+        Command(
+            name="duplicate",
+            docstring="duplicate args",
+            arguments=[
+                Argument(name="arg1", type="string", description="first", required=True),
+                Argument(name="arg1", type="string", description="duplicate", required=True),
+            ],
+        )
+
+
+def test_argument_name_patterns():
+    """Test valid and invalid argument name patterns."""
+    # Valid names including single characters
+    valid_names = ["a", "x", "n", "simple", "with_underscore", "with-dash", "with123numbers", "_starts_with_underscore"]
+
+    for name in valid_names:
+        command = Command(
+            name="test",
+            docstring="test",
+            arguments=[Argument(name=name, type="string", description="test", required=True)],
+        )
+        assert command.arguments[0].name == name
+
+    # Invalid names
+    invalid_names = ["123starts_with_number", ""]
+
+    for name in invalid_names:
+        with pytest.raises(ValueError, match="Invalid argument name"):
+            Command(
+                name="test",
+                docstring="test",
+                arguments=[Argument(name=name, type="string", description="test", required=True)],
+            )
+
+
+def test_signature_argument_consistency():
+    """Test that signatures and arguments must be consistent."""
+    # Missing argument in signature
+    with pytest.raises(ValueError, match="Missing argument.*in signature"):
+        Command(
+            name="missing_arg",
+            signature="missing_arg <existing_arg>",
+            docstring="missing argument in signature",
+            arguments=[
+                Argument(name="existing_arg", type="string", description="exists", required=True),
+                Argument(name="missing_arg", type="string", description="not in signature", required=True),
+            ],
+        )
+
+    # Extra argument in signature
+    with pytest.raises(ValueError, match="Argument names.*do not match"):
+        Command(
+            name="extra_arg",
+            signature="extra_arg <arg1> <extra>",
+            docstring="extra argument in signature",
+            arguments=[Argument(name="arg1", type="string", description="exists", required=True)],
+        )
+
+
+def test_function_calling_tool_generation():
+    """Test OpenAI function calling tool generation."""
+    command = Command(
+        name="test_function",
+        docstring="A test function for OpenAI",
+        arguments=[
+            Argument(name="required_arg", type="string", description="Required string argument", required=True),
+            Argument(
+                name="enum_arg", type="string", description="Enum argument", required=True, enum=["option1", "option2"]
+            ),
+            Argument(name="optional_arg", type="integer", description="Optional integer argument", required=False),
+        ],
+    )
+
+    tool = command.get_function_calling_tool()
+
+    assert tool["type"] == "function"
+    assert tool["function"]["name"] == "test_function"
+    assert tool["function"]["description"] == "A test function for OpenAI"
+
+    properties = tool["function"]["parameters"]["properties"]
+    assert properties["required_arg"]["type"] == "string"
+    assert properties["optional_arg"]["type"] == "integer"
+    assert properties["enum_arg"]["enum"] == ["option1", "option2"]
+
+    required = tool["function"]["parameters"]["required"]
+    assert "required_arg" in required
+    assert "enum_arg" in required
+    assert "optional_arg" not in required
+
+
+def test_multiline_command():
+    """Test multi-line commands with end markers."""
+    command = Command(
+        name="edit",
+        signature="edit <filename>",
+        docstring="Edit a file with multi-line content",
+        end_name="EOF",
+        arguments=[Argument(name="filename", type="string", description="file to edit", required=True)],
+    )
+
+    assert command.invoke_format == "edit {filename}"
+    assert command.end_name == "EOF"
+
+
+def test_custom_argument_format():
+    """Test custom argument formatting."""
+    command = Command(
+        name="custom_format",
+        docstring="Test custom argument formatting",
+        arguments=[
+            Argument(
+                name="arg1",
+                type="string",
+                description="Custom formatted argument",
+                required=True,
+                argument_format="--{value}",
+            )
+        ],
+    )
+
+    assert command.arguments[0].argument_format == "--{value}"
+    assert command.invoke_format == "custom_format {arg1} "

package/python/tests/test_utils.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from sweagent import REPO_ROOT
+from sweagent.utils.config import _convert_path_to_abspath, _convert_paths_to_abspath
+
+
+def test_convert_path_to_abspath():
+    assert _convert_path_to_abspath("sadf") == REPO_ROOT / "sadf"
+    assert _convert_path_to_abspath("/sadf") == Path("/sadf")
+
+
+def test_convert_paths_to_abspath():
+    assert _convert_paths_to_abspath([Path("sadf"), Path("/sadf")]) == [REPO_ROOT / "sadf", Path("/sadf")]

package/python/tests/tools/conftest.py

@@ -0,0 +1,12 @@
+from unittest import mock
+
+import pytest
+
+
+@pytest.fixture
+def with_tmp_env_file(tmp_path):
+    env_file = tmp_path / ".swe-agent-env"
+    env_file.write_text("{}")
+    with mock.patch.dict("os.environ", {"SWE_AGENT_ENV_FILE": str(env_file)}, clear=True):
+        yield env_file
+    env_file.unlink()