@elizaos/sweagent-root 2.0.0-alpha

package/python/sweagent/inspector/server.py

@@ -0,0 +1,354 @@
+from __future__ import annotations
+
+import http.server
+import json
+import os
+import socketserver
+from argparse import ArgumentParser
+from functools import partial
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+def add_problem_statement(content):
+    """The problem statement is the first 'user' message in the history.
+
+    We'll prepend the trajectory with the problem statement.
+    """
+    problem_statement = ""
+    for item in content["history"]:
+        if item["role"] == "user":
+            problem_statement = item["content"]
+            break
+    if problem_statement:
+        content["trajectory"].insert(
+            0,
+            {
+                "thought": "",
+                "action": "",
+                "response": "",
+                "observation": problem_statement,
+                "messages": [{"role": "system", "content": "Problem Statement placeholder"}],
+            },
+        )
+    return content
+
+
+def append_exit(content):
+    exit_status = content.get("info", {}).get("exit_status", None)
+    if exit_status is None:
+        return content
+
+    if exit_status.startswith("submitted"):
+        if "submission" in content["info"]:
+            content["trajectory"].append(
+                {
+                    "thought": "Submitting solution",
+                    "action": "Model Submission",
+                    "response": "Submitting solution",
+                    "observation": content["info"]["submission"],
+                    "messages": [{"role": "system", "content": f"Submission generated - {exit_status}"}],
+                }
+            )
+        else:
+            msg = "No submission in history or info"
+            raise ValueError(msg)
+    return content
+
+
+def append_patch(instance_id, content, patches, patch_type):
+    if content.get("info", {}).get("exit_status", None) is not None:
+        if instance_id in patches:
+            content["trajectory"].append(
+                {
+                    "thought": f"Showing {patch_type} patch",
+                    "response": f"Showing {patch_type} patch",
+                    "action": f"{patch_type} Patch",
+                    "observation": patches[instance_id],
+                }
+            )
+    return content
+
+
+def append_results(traj_path: Path, instance_id: str, content, results, results_file):
+    stats: list[str] = []
+    model_stats = {}
+    if traj_path.exists():
+        data = json.loads(traj_path.read_text())
+        info = data.get("info", {})
+        model_stats = info.get("model_stats", {})
+
+    # Build stats section
+    exit_status = info.get("exit_status", "N/A")
+    instance_cost = model_stats.get("instance_cost", None)
+    instance_cost = f"{instance_cost:.2f}" if instance_cost is not None else "N/A"
+    tokens_sent = model_stats.get("tokens_sent", None)
+    tokens_sent = f"{tokens_sent:,}" if tokens_sent is not None else "N/A"
+    tokens_received = model_stats.get("tokens_received", None)
+    tokens_received = f"{tokens_received:,}" if tokens_received is not None else "N/A"
+    api_calls = model_stats.get("api_calls", None)
+    api_calls = f"{api_calls:,}" if api_calls is not None else "N/A"
+
+    stats.append("**** Run Stats ****")
+    stats.append(f"Exit Status: {exit_status}")
+    stats.append(f"Instance Cost: ${instance_cost}")
+    stats.append(f"Tokens Sent: {tokens_sent}")
+    stats.append(f"Tokens Received: {tokens_received}")
+    stats.append(f"API Calls: {api_calls}\n")
+
+    # Build status section
+    status = []
+    if results is None:
+        status.append("Evaluation results not found")
+    elif "completed_ids" in results and "submitted_ids" in results and "resolved_ids" in results:
+        is_completed = instance_id in results["completed_ids"]
+        is_submitted = instance_id in results["submitted_ids"]
+        is_resolved = instance_id in results["resolved_ids"]
+
+        status.append("**** Statuses ****")
+        status.append(f"  {'✅' if is_completed else '❌'} Completed (The agent successfully ran)")
+        status.append(f"  {'✅' if is_submitted else '❌'} Submitted (The agent successfully submitted a pull request)")
+        status.append(
+            f"  {'✅' if is_resolved else '❌'} Resolved (The pull request {'' if is_resolved else 'has not '}"
+            "successfully resolved the issue during eval)"
+        )
+    else:
+        status.append("Results format not recognized")
+
+    if status == []:
+        status.append("Instance not found in results")
+    else:
+        status.append("---------------------------")
+        status.append(
+            "Note that the evaluation results here may not be accurate or up to date, since they are computed separately from the agent run itself."
+        )
+        status.append(f"Check {results_file} for the most accurate evaluation results.")
+        status.append("")
+        status.append(f"Instance ID: {instance_id}")
+
+    # Add evaluation report as first and last items in trajectory
+    eval_report = {
+        "thought": "Evaluation Report",
+        "action": "Showing evaluation results",
+        "response": "Showing evaluation results",
+        "observation": "\n".join([*stats, *status]),
+        "messages": [{"role": "system", "content": "Showing evaluation results and statistics"}],
+    }
+
+    if not content.get("trajectory"):
+        content["trajectory"] = []
+    content["trajectory"].insert(0, eval_report)
+    content["trajectory"].append(eval_report)
+    return content
+
+
+def get_action_summary(content):
+    out = ""
+    i = 0
+    for item in content["history"]:
+        if item["role"] != "assistant":
+            continue
+        if item.get("is_demo"):
+            continue
+        i += 1
+        try:
+            action = item["action"]
+        except KeyError:
+            print(f"No action for step {i}")
+            print(item)
+            raise
+        if len(action) > 70:
+            action = action[:67] + "..."
+        out += f"Step {i}: {action}\n"
+    return out
+
+
+def load_content(file_name, gold_patches, test_patches) -> dict[str, Any]:
+    with open(file_name) as infile:
+        content = json.load(infile)
+    results_file = Path(file_name).parent / "results.json"
+    results = load_results(results_file)
+
+    content = add_problem_statement(content)
+    content = append_exit(content)
+    content = append_patch(Path(file_name).stem, content, gold_patches, "Gold")
+    content = append_patch(Path(file_name).stem, content, test_patches, "Test")
+    content["history"].insert(0, {"role": "Action Summary", "content": get_action_summary(content)})
+    return append_results(
+        Path(file_name),
+        Path(file_name).stem,
+        content,
+        results,
+        results_file,
+    )
+
+
+def load_results(results_path: Path) -> dict[str, Any] | None:
+    """Load results from results.json.
+
+    If file is not found, return None.
+    """
+    if not results_path.exists():
+        return None
+    with open(results_path) as infile:
+        results = json.load(infile)
+    # Different versions of the code used "not_generated" or "no_generation".
+    # Let's standardize this here
+    if "no_generation" in results:
+        results["not_generated"] = results["no_generation"]
+        del results["no_generation"]
+    return results
+
+
+def get_status(traj_path) -> str:
+    """Return results emoji for single trajectory"""
+    results = load_results(Path(traj_path).parent / "results.json")
+    info = json.loads(Path(traj_path).read_text()).get("info", {})
+    n_steps = info.get("model_stats", {}).get("api_calls", "N/A")
+    exit_status = info.get("exit_status", "N/A")
+    exit_status_str = f" ({exit_status} after {n_steps} steps)"
+    instance_id = Path(traj_path).stem
+    if results is None:
+        return f"❓ {exit_status_str}"
+    elif instance_id in results["resolved_ids"]:
+        return "✅"
+    else:
+        return f"❌ {exit_status_str}"
+
+
+class Handler(http.server.SimpleHTTPRequestHandler):
+    file_mod_times = {}  # Dictionary to keep track of file modification times
+
+    def __init__(self, *args, **kwargs):
+        self.gold_patches = {}
+        self.test_patches = {}
+        if "gold_patches" in kwargs:
+            self.gold_patches = kwargs.pop("gold_patches")
+        if "test_patches" in kwargs:
+            self.test_patches = kwargs.pop("test_patches")
+        self.traj_dir = kwargs.pop("directory", ".")  # Extract directory
+        super().__init__(*args, **kwargs)
+
+    def serve_directory_info(self):
+        self.send_response(200)
+        self.send_header("Content-type", "application/json")
+        self.end_headers()
+        self.wfile.write(json.dumps({"directory": self.traj_dir}).encode())
+
+    def serve_file_content(self, file_path):
+        try:
+            content = load_content(
+                Path(self.traj_dir) / file_path,
+                self.gold_patches,
+                self.test_patches,
+            )
+            self.send_response(200)
+            self.send_header("Content-type", "text/plain")
+            self.end_headers()
+            self.wfile.write(json.dumps(content).encode())
+        except FileNotFoundError:
+            self.send_error(404, f"File {file_path} not found")
+
+    def do_GET(self):
+        if self.path == "/directory_info":
+            self.serve_directory_info()
+        elif self.path.startswith("/files"):
+            self.handle_files_request()
+        elif self.path.startswith("/trajectory/"):
+            file_path = self.path[len("/trajectory/") :]
+            self.serve_file_content(file_path)
+        elif self.path.startswith("/check_update"):
+            self.check_for_updates()
+        else:
+            super().do_GET()
+
+    def handle_files_request(self):
+        self.send_response(200)
+        self.send_header("Content-type", "application/json")
+        self.end_headers()
+        files = sorted(
+            (
+                str(file.relative_to(Path(self.traj_dir))) + " " * 4 + get_status(file)
+                for file in Path(self.traj_dir).glob("**/*.traj")
+            ),
+            key=lambda x: str(Path(self.traj_dir) / x),
+            reverse=True,
+        )
+        self.wfile.write(json.dumps(files).encode())
+
+    def check_for_updates(self):
+        current_mod_times = {str(file): file.stat().st_mtime for file in Path(self.traj_dir).glob("**/*.traj")}
+        if current_mod_times != Handler.file_mod_times:
+            Handler.file_mod_times = current_mod_times
+            self.send_response(200)  # Send response that there's an update
+        else:
+            self.send_response(204)  # Send no content response if no update
+        self.end_headers()
+
+    def end_headers(self):
+        self.send_header("Access-Control-Allow-Origin", "*")
+        super().end_headers()
+
+
+def main(data_path, directory, port):
+    data = []
+    if data_path is not None:
+        if data_path.endswith(".jsonl"):
+            data = [json.loads(x) for x in Path(data_path).read_text().splitlines(keepends=True)]
+        elif data_path.endswith(".json"):
+            with open(data_path) as f:
+                data = json.load(f)
+    elif "args.yaml" in os.listdir(directory):
+        with open(Path(directory) / "args.yaml") as file:
+            args = yaml.safe_load(file)
+        if "environment" in args and "data_path" in args["environment"]:
+            data_path = Path(__file__).parent.parent / args["environment"]["data_path"]
+            if data_path.exists:
+                with open(data_path) as f:
+                    data = json.load(f)
+
+    gold_patches = {d["instance_id"]: d["patch"] if "patch" in d else None for d in data}
+    test_patches = {d["instance_id"]: d["test_patch"] if "test_patch" in d else None for d in data}
+
+    handler_with_directory = partial(
+        Handler,
+        directory=directory,
+        gold_patches=gold_patches,
+        test_patches=test_patches,
+    )
+    try:
+        with socketserver.TCPServer(("", port), handler_with_directory) as httpd:
+            print(f"Serving at http://localhost:{port}")
+            httpd.serve_forever()
+    except OSError as e:
+        if e.errno == 48:
+            print(f"ERROR: Port ({port}) is already in use. Try another port with the --port flag.")
+        else:
+            raise e
+
+
+def get_parser():
+    parser = ArgumentParser()
+    parser.add_argument(
+        "--data_path",
+        type=str,
+        help="Path to dataset that was used for the trajectories. Necessary to display gold patches.",
+    )
+    parser.add_argument("--directory", type=str, help="Directory to serve", default=os.getcwd(), nargs="?")
+    parser.add_argument("--port", type=int, help="Port to serve", default=8000)
+    return parser
+
+
+def run_from_cli(args: list[str] | None = None):
+    # Hack to make sure all the templates and all are found
+    parsed_args = get_parser().parse_args(args)
+    # convert directory, relative to the absolute path
+    parsed_args.directory = str(Path(parsed_args.directory).resolve().absolute())
+    os.chdir(Path(__file__).parent)
+    main(**vars(parsed_args))
+
+
+if __name__ == "__main__":
+    run_from_cli()

package/python/sweagent/inspector/static.py

@@ -0,0 +1,169 @@
+from __future__ import annotations
+
+import json
+import logging
+import traceback
+from argparse import ArgumentParser
+from pathlib import Path
+
+import yaml
+from tqdm.auto import tqdm
+
+try:
+    from .server import load_content
+except ImportError:
+    from server import load_content
+
+
+logger = logging.getLogger(__name__)
+logging.getLogger("simple_parsing").setLevel(logging.INFO)
+
+
+TEMPLATE = """
+<html>
+<head>
+    <title>Trajectory Viewer</title>
+    <style>
+    {style_sheet}
+    </style>
+</head>
+<body>
+    <div class="container">
+        {file_path_tree}
+        <h2>Conversation History</h2>
+        <pre id="fileContent">{file_content}</pre>
+    </div>
+</body>
+</html>
+"""
+
+try:
+    with open(Path(__file__).parent / "style.css") as infile:
+        STYLE_SHEET = infile.read()
+except Exception as e:
+    style_file = Path(__file__).parent / "style.css"
+    logger.error(f"Failed to load style sheet from {style_file}: {traceback.format_exc()}")
+    raise e
+
+
+def _load_file(file_name, gold_patches, test_patches):
+    try:
+        role_map = {
+            "user": "Computer",
+            "assistant": "SWE-Agent",
+            "subroutine": "SWE-Agent subroutine",
+            "default": "Default",
+            "system": "System",
+            "demo": "Demonstration",
+        }
+        content = load_content(file_name, gold_patches, test_patches)
+        if "history" in content and isinstance(content["history"], list):
+            history_content = ""
+            for index, item in enumerate(content["history"]):
+                item_content = item.get("content", "").replace("<", "&lt;").replace(">", "&gt;")
+                if item.get("agent") and item["agent"] != "primary":
+                    role_class = "subroutine"
+                else:
+                    role_class = item.get("role", "default").lower().replace(" ", "-")
+                element_id = f"historyItem{index}"
+                role_name = role_map.get(item.get("role", ""), item.get("role", ""))
+                history_content += (
+                    f"""<div class="history-item {role_class}" id="{element_id}">"""
+                    f"""<div class="role-bar {role_class}"><strong><span>{role_name}</span></strong></div>"""
+                    f"""<div class="content-container">"""
+                    f"""<pre>{item_content}</pre>"""
+                    f"""</div>"""
+                    f"""<div class="shadow"></div>"""
+                    f"""</div>"""
+                )
+            return history_content
+        else:
+            return "No history content found."
+    except Exception:
+        return f"Error loading content. {traceback.format_exc()}"
+
+
+def _make_file_path_tree(file_path):
+    path_parts = file_path.split("/")
+    relevant_parts = path_parts[-3:]
+    html_string = '<div class="filepath">\n'
+    for part in relevant_parts:
+        html_string += f'<div class="part">{part}</div>\n'
+    html_string += "</div>"
+    return html_string
+
+
+def save_static_viewer(file_path):
+    if not isinstance(file_path, Path):
+        file_path = Path(file_path)
+    data = []
+    if "args.yaml" in list(map(lambda x: x.name, file_path.parent.iterdir())):
+        args = yaml.safe_load(Path(file_path.parent / "args.yaml").read_text())
+        if "environment" in args and "data_path" in args["environment"]:
+            data_path = Path(__file__).parent.parent / args["environment"]["data_path"]
+            if data_path.exists():
+                with open(data_path) as f:
+                    data = json.load(f)
+            if not isinstance(data, list) or not data or "patch" not in data[0] or "test_patch" not in data[0]:
+                data = []
+    gold_patches = {x["instance_id"]: x["patch"] for x in data}
+    test_patches = {x["instance_id"]: x["test_patch"] for x in data}
+    content = _load_file(file_path, gold_patches, test_patches)
+    file_path_tree = _make_file_path_tree(file_path.absolute().as_posix())
+    icons_path = Path(__file__).parent / "icons"
+    relative_icons_path = find_relative_path(file_path, icons_path)
+    style_sheet = STYLE_SHEET.replace("url('icons/", f"url('{relative_icons_path.as_posix()}/").replace(
+        'url("icons/',
+        f'url("{relative_icons_path.as_posix()}/',
+    )
+    data = TEMPLATE.format(file_content=content, style_sheet=style_sheet, file_path_tree=file_path_tree)
+    output_file = file_path.with_suffix(".html")
+    with open(output_file, "w+") as outfile:
+        print(data, file=outfile)
+    logger.info(f"Saved static viewer to {output_file}")
+
+
+def find_relative_path(from_path, to_path):
+    # Convert paths to absolute for uniformity
+    from_path = from_path.resolve()
+    to_path = to_path.resolve()
+    if from_path.is_file():
+        from_path = from_path.parent
+    if to_path.is_file():
+        to_path = to_path.parent
+    if not from_path.is_dir() or not to_path.is_dir():
+        msg = f"Both from_path and to_path must be directories, but got {from_path} and {to_path}"
+        raise ValueError(msg)
+
+    # Identify the common ancestor and the parts of each path beyond it
+    common_parts = 0
+    for from_part, to_part in zip(from_path.parts, to_path.parts):
+        if from_part != to_part:
+            break
+        common_parts += 1
+
+    # Calculate the '../' needed to get back from from_path to the common ancestor
+    back_to_ancestor = [".."] * (len(from_path.parts) - common_parts)
+
+    # Direct path from common ancestor to to_path
+    to_target = to_path.parts[common_parts:]
+
+    # Combine to get the relative path
+    return Path(*back_to_ancestor, *to_target)
+
+
+def save_all_trajectories(directory):
+    if not isinstance(directory, Path):
+        directory = Path(directory)
+    all_files = list(directory.glob("**/*.traj"))
+    logger.info(f"Found {len(all_files)} trajectory files in {directory}")
+    for file_path in tqdm(all_files, desc="Saving static viewers"):
+        save_static_viewer(file_path)
+    logger.info(f"Saved static viewers for all trajectories in {args.directory}")
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser()
+    parser.add_argument("directory", type=str, help="Directory containing trajectory files")
+    args = parser.parse_args()
+    save_all_trajectories(args.directory)