@tonyclaw/eval-harness 0.2.18

package/src/eval/runners/codex.py

@@ -0,0 +1,412 @@
+from __future__ import annotations
+
+import json
+import os
+import re
+import shutil
+from pathlib import Path
+from typing import Any
+
+from .base import DEFAULT_AGENT_PROMPT, _command_set_named_arg
+from .opencode import OpenCodeRunner
+
+CODEX_EXECUTABLE_ENV_KEYS = (
+    "OEVAL_CODEX_EXECUTABLE",
+    "CODEX_EXECUTABLE",
+    "CODEX_CLI",
+    "CODEX_EXE",
+)
+
+
+class CodexCliRunner(OpenCodeRunner):
+    """Codex CLI agent runner adapter."""
+
+    @property
+    def name(self) -> str:
+        return "codex"
+
+    @property
+    def binary(self) -> str:
+        return "codex"
+
+    @property
+    def config_key(self) -> str:
+        return "codex"
+
+    # -- command building
+
+    def default_command(self, config: dict[str, Any]) -> list[str]:
+        executable = self.executable(config)
+        return [
+            executable,
+            "exec",
+            "--json",
+            "--skip-git-repo-check",
+            "--dangerously-bypass-approvals-and-sandbox",
+            "--cd",
+            "{workspace}",
+            DEFAULT_AGENT_PROMPT,
+        ]
+
+    def executable(self, config: dict[str, Any]) -> str:
+        for value in (
+            (config.get("agent") or {}).get("executable") if isinstance(config.get("agent"), dict) else None,
+            (config.get("codex") or {}).get("executable") if isinstance(config.get("codex"), dict) else None,
+            (config.get("execution") or {}).get("codex_executable") if isinstance(config.get("execution"), dict) else None,
+            ((config.get("execution") or {}).get("codex") or {}).get("executable")
+            if isinstance(config.get("execution"), dict) and isinstance((config.get("execution") or {}).get("codex"), dict)
+            else None,
+        ):
+            text = str(value or "").strip()
+            if text:
+                return text
+        return discover_codex_executable() or "codex"
+
+    def format_flags(self) -> list[str]:
+        return ["--json"]
+
+    def command_with_variant(self, command: list[str], variant: str) -> list[str]:
+        return command
+
+    def variant_cli_flag(self) -> str:
+        return ""
+
+    def variant_from_command(self, command: list[str]) -> str:
+        return ""
+
+    def default_worker_tools(self, config: dict[str, Any] | None = None) -> list[str]:
+        return [self.executable(config or {})]
+
+    def command_with_model(self, command: list[str], model: str) -> list[str]:
+        model = _codex_cli_model_name(model)
+        for item in command:
+            text = str(item)
+            if text == "--model" or text.startswith("--model="):
+                return _command_set_named_arg(command, "--model", model)
+        if len(command) >= 3 and str(command[1]).lower() == "exec":
+            value_flags = {
+                "--ask-for-approval",
+                "--cd",
+                "--config",
+                "--model",
+                "--sandbox",
+                "-C",
+                "-c",
+                "-m",
+            }
+            previous = str(command[-2]) if len(command) >= 2 else ""
+            last = str(command[-1])
+            if last and not last.startswith("-") and previous not in value_flags:
+                return [*command[:-1], "--model", model, command[-1]]
+        return [*command, "--model", model]
+
+    # -- log directory
+
+    def log_dir_name(self) -> str:
+        return "codex"
+
+    # -- asset installation
+
+    def default_asset_installs(self) -> list[dict[str, Any]]:
+        return [
+            {
+                "source": "work/work/skills/*",
+                "target": ".codex/skills",
+                "kind": "skills",
+                "entries": "directories",
+                "required": False,
+                "clean": True,
+            },
+            {
+                "source": "work/work/skills/versions.json",
+                "target": ".codex/skills",
+                "kind": "skill-manifest",
+                "entries": "files",
+                "required": False,
+                "clean": False,
+            },
+            {
+                "source": "work/skills/*",
+                "target": ".codex/skills",
+                "kind": "skills-flat-fallback",
+                "entries": "directories",
+                "required": False,
+                "clean": False,
+            },
+            {
+                "source": "work/skills/versions.json",
+                "target": ".codex/skills",
+                "kind": "skill-manifest-flat-fallback",
+                "entries": "files",
+                "required": False,
+                "clean": False,
+            },
+            {
+                "source": "work/work/skills/*.md",
+                "target": ".codex/agents",
+                "kind": "agents",
+                "required": False,
+                "clean": True,
+            },
+            {
+                "source": "work/skills/*.md",
+                "target": ".codex/agents",
+                "kind": "agents-flat-fallback",
+                "required": False,
+                "clean": False,
+            },
+        ]
+
+    # -- runtime profile seeding
+
+    def runtime_profile_candidates(
+        self, base_env: dict[str, str], runtime: dict[str, str]
+    ) -> list[tuple[Path | None, Path, str]]:
+        from ..cli import existing_path
+
+        home = Path(runtime["home_dir"])
+        appdata = Path(runtime.get("appdata_dir", home / "AppData" / "Roaming"))
+        localappdata = Path(runtime.get("localappdata_dir", home / "AppData" / "Local"))
+        config_dir = Path(runtime["config_dir"])
+        data_dir = Path(runtime["data_dir"])
+
+        return [
+            (existing_path(base_env.get("CODEX_HOME")), home / ".codex", "CODEX_HOME"),
+            (existing_path(base_env.get("USERPROFILE"), ".codex"), home / ".codex", "USERPROFILE/.codex"),
+            (existing_path(base_env.get("HOME"), ".codex"), home / ".codex", "HOME/.codex"),
+            (existing_path(base_env.get("USERPROFILE"), ".config", "codex"), config_dir / "codex", "USERPROFILE/.config/codex"),
+            (existing_path(base_env.get("USERPROFILE"), ".local", "share", "codex"), data_dir / "codex", "USERPROFILE/.local/share/codex"),
+            (existing_path(base_env.get("USERPROFILE"), ".cache", "codex"), Path(runtime["cache_dir"]) / "codex", "USERPROFILE/.cache/codex"),
+            (existing_path(base_env.get("APPDATA"), "codex"), appdata / "codex", "APPDATA/codex"),
+            (existing_path(base_env.get("LOCALAPPDATA"), "codex"), localappdata / "codex", "LOCALAPPDATA/codex"),
+            (existing_path(base_env.get("XDG_CONFIG_HOME"), "codex"), config_dir / "codex", "XDG_CONFIG_HOME/codex"),
+            (existing_path(base_env.get("XDG_DATA_HOME"), "codex"), data_dir / "codex", "XDG_DATA_HOME/codex"),
+        ]
+
+    # -- provider config
+
+    def write_agent_config(self, manifest: dict[str, Any]) -> str | None:
+        runtime = manifest.get("runtime") or {}
+        home = runtime.get("home_dir")
+        if not home:
+            return None
+        config_dir = Path(home) / ".codex"
+        config_dir.mkdir(parents=True, exist_ok=True)
+        config_path = config_dir / "config.toml"
+
+        from ..cli import execution_agent_provider  # local to avoid circular import
+
+        execution = manifest.get("execution", {})
+        provider = execution_agent_provider(execution, self)
+        if not isinstance(provider, dict):
+            return str(config_path)
+        provider_id = str(provider.get("id", "agent-inspector")).strip() or "agent-inspector"
+        if provider_id != "agent-inspector":
+            return str(config_path)
+
+        base_url = str(provider.get("baseURL", "http://127.0.0.1:25947/proxy")).strip()
+        if not base_url:
+            base_url = "http://127.0.0.1:25947/proxy"
+        base_url = _codex_provider_base_url(base_url)
+        provider_name = str(provider.get("name", "Agent Inspector")).strip() or "Agent Inspector"
+        api_key = str(provider.get("apiKey", "") or "")
+        wire_api = _codex_provider_wire_api(provider)
+
+        existing = config_path.read_text(encoding="utf-8") if config_path.exists() else ""
+        preserved = _remove_codex_inspector_config(existing)
+        top_level_block = [
+            "# Managed by eval-harness for Agent Inspector runs.",
+            f'model_provider = "{_toml_escape(provider_id)}"',
+        ]
+        provider_block = [
+            "",
+            "# Managed by eval-harness for Agent Inspector runs.",
+            f'[model_providers."{_toml_escape(provider_id)}"]',
+            f'name = "{_toml_escape(provider_name)}"',
+            f'base_url = "{_toml_escape(base_url)}"',
+            f'wire_api = "{_toml_escape(wire_api)}"',
+        ]
+        if api_key:
+            provider_block.append(f'experimental_bearer_token = "{_toml_escape(api_key)}"')
+        text = _insert_codex_top_level_config(preserved, top_level_block)
+        text = text.rstrip() + "\n" + "\n".join(provider_block) + "\n"
+        config_path.write_text(text, encoding="utf-8")
+        return str(config_path)
+
+    # -- presentation
+
+    def extract_sessions_from_text(
+        self, text: str, *, inspector_base_url: str = "http://127.0.0.1:25947"
+    ) -> dict[str, Any]:
+        summary = super().extract_sessions_from_text(text, inspector_base_url=inspector_base_url)
+        thread_ids: list[str] = []
+        for raw_line in text.splitlines():
+            line = raw_line.strip()
+            if not line.startswith("{"):
+                continue
+            try:
+                payload = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if not isinstance(payload, dict):
+                continue
+            thread_id = str(payload.get("thread_id") or "").strip()
+            if thread_id and thread_id not in thread_ids:
+                thread_ids.append(thread_id)
+        summary["codex_thread_ids"] = thread_ids
+        summary["primary_codex_thread_id"] = thread_ids[0] if thread_ids else None
+        return summary
+
+    def render_sessions_md(self, summary: dict[str, Any]) -> str:
+        text = super().render_sessions_md(summary)
+        text = text.replace("# Codex Sessions", "# Codex CLI Sessions", 1)
+        lines = [text.rstrip()]
+        thread_ids = summary.get("codex_thread_ids") or []
+        if thread_ids:
+            lines.extend(["", "## Codex Threads", ""])
+            for thread_id in thread_ids:
+                lines.append(f"- `{thread_id}`")
+        return "\n".join(lines) + "\n"
+
+
+def _codex_cli_model_name(model: str) -> str:
+    text = str(model or "").strip()
+    prefix = "agent-inspector/"
+    if text.startswith(prefix):
+        text = text[len(prefix) :]
+    return _codex_inspector_api_model_name(text)
+
+
+def discover_codex_executable(env: dict[str, str] | None = None) -> str:
+    """Find Codex CLI without relying on an interactive shell PATH."""
+    env_map = os.environ if env is None else env
+    for key in CODEX_EXECUTABLE_ENV_KEYS:
+        found = _existing_executable(env_map.get(key))
+        if found:
+            return found
+
+    if os.name == "nt":
+        candidates = _windows_codex_install_candidates(env_map)
+        if candidates:
+            return str(candidates[0])
+
+    for name in ("codex", "codex.exe"):
+        found = shutil.which(name)
+        if found:
+            return found
+    return ""
+
+
+def _existing_executable(value: Any) -> str:
+    text = str(value or "").strip().strip('"')
+    if not text:
+        return ""
+    path = Path(text)
+    if path.is_file():
+        return str(path)
+    found = shutil.which(text)
+    return found or ""
+
+
+def _windows_codex_install_candidates(env: dict[str, str]) -> list[Path]:
+    roots: list[Path] = []
+    for raw in (
+        env.get("LOCALAPPDATA"),
+        str(Path(env["USERPROFILE"]) / "AppData" / "Local") if env.get("USERPROFILE") else "",
+        str(Path.home() / "AppData" / "Local"),
+    ):
+        text = str(raw or "").strip()
+        if not text:
+            continue
+        root = Path(text) / "OpenAI" / "Codex" / "bin"
+        if root not in roots:
+            roots.append(root)
+
+    candidates: list[Path] = []
+    for root in roots:
+        if not root.exists():
+            continue
+        candidates.extend(root.glob("codex.exe"))
+        candidates.extend(root.glob("*/codex.exe"))
+
+    unique: dict[str, Path] = {}
+    for path in candidates:
+        if path.is_file():
+            unique[str(path).lower()] = path
+
+    def sort_key(path: Path) -> tuple[float, str]:
+        try:
+            mtime = path.stat().st_mtime
+        except OSError:
+            mtime = 0.0
+        return (mtime, str(path))
+
+    return sorted(unique.values(), key=sort_key, reverse=True)
+
+
+def _codex_inspector_api_model_name(model: str) -> str:
+    text = str(model or "").strip()
+    if re.match(r"(?i)^minimax\s+m", text):
+        return re.sub(r"\s+", "-", text)
+    return text
+
+
+def _remove_codex_inspector_config(text: str) -> str:
+    lines = text.splitlines()
+    result: list[str] = []
+    skip = False
+    for line in lines:
+        stripped = line.strip()
+        if stripped == "# Managed by eval-harness for Agent Inspector runs.":
+            continue
+        if stripped == 'model_provider = "agent-inspector"':
+            continue
+        if stripped == '[model_providers."agent-inspector"]' or stripped == "[model_providers.agent-inspector]":
+            skip = True
+            continue
+        if skip and stripped.startswith("[") and stripped.endswith("]"):
+            skip = False
+        if not skip:
+            result.append(line)
+    return "\n".join(result)
+
+
+def _insert_codex_top_level_config(text: str, block: list[str]) -> str:
+    lines = text.rstrip().splitlines() if text.strip() else []
+    first_table = len(lines)
+    for index, line in enumerate(lines):
+        stripped = line.strip()
+        if stripped.startswith("[") and stripped.endswith("]"):
+            first_table = index
+            break
+
+    before = lines[:first_table]
+    after = lines[first_table:]
+    result: list[str] = []
+    result.extend(before)
+    if result and result[-1].strip():
+        result.append("")
+    result.extend(block)
+    if after:
+        result.append("")
+        result.extend(after)
+    return "\n".join(result)
+
+
+def _codex_provider_wire_api(provider: dict[str, Any]) -> str:
+    for key in ("codexWireApi", "codex_wire_api", "wireApi", "wire_api"):
+        value = str(provider.get(key, "") or "").strip()
+        if value:
+            return value
+    return "responses"
+
+
+def _codex_provider_base_url(base_url: str) -> str:
+    value = base_url.rstrip("/")
+    if value.endswith("/v1"):
+        return value
+    return f"{value}/v1"
+
+
+def _toml_escape(value: str) -> str:
+    return value.replace("\\", "\\\\").replace('"', '\\"')