eval-harness 0.2.18

package/src/eval/runners/claude.py

@@ -0,0 +1,516 @@
+from __future__ import annotations
+
+import json
+import re
+import urllib.parse
+from pathlib import Path
+from typing import Any
+
+from .base import DEFAULT_AGENT_PROMPT, _command_get_named_arg, _command_set_named_arg, normalize_agent_value
+from .opencode import OpenCodeRunner
+
+
+class ClaudeCodeRunner(OpenCodeRunner):
+    """Claude Code CLI agent runner adapter."""
+
+    @property
+    def name(self) -> str:
+        return "claude"
+
+    @property
+    def binary(self) -> str:
+        return "claude"
+
+    @property
+    def config_key(self) -> str:
+        return "claude"
+
+    # -- command building
+
+    def default_command(self, config: dict[str, Any]) -> list[str]:
+        executable = self.executable(config)
+        return [
+            executable,
+            "-p",
+            "--output-format",
+            "stream-json",
+            "--verbose",
+            "--dangerously-skip-permissions",
+            DEFAULT_AGENT_PROMPT,
+        ]
+
+    def executable(self, config: dict[str, Any]) -> str:
+        for value in (
+            (config.get("agent") or {}).get("executable") if isinstance(config.get("agent"), dict) else None,
+            (config.get("claude") or {}).get("executable") if isinstance(config.get("claude"), dict) else None,
+            (config.get("execution") or {}).get("claude_executable") if isinstance(config.get("execution"), dict) else None,
+            ((config.get("execution") or {}).get("claude") or {}).get("executable")
+            if isinstance(config.get("execution"), dict) and isinstance((config.get("execution") or {}).get("claude"), dict)
+            else None,
+        ):
+            text = str(value or "").strip()
+            if text:
+                return text
+        return "claude"
+
+    def format_flags(self) -> list[str]:
+        return ["--output-format", "stream-json"]
+
+    def command_with_model(self, command: list[str], model: str) -> list[str]:
+        model = _claude_model_name(model)
+        if _command_get_named_arg(command, "--model"):
+            return _command_set_named_arg(command, "--model", model)
+        if _command_get_named_arg(command, "-m"):
+            return _command_set_named_arg(command, "-m", model)
+        if command and _is_claude_executable(str(command[0])):
+            previous = str(command[-2]) if len(command) >= 2 else ""
+            last = str(command[-1])
+            if last and not last.startswith("-") and previous not in _CLAUDE_VALUE_FLAGS:
+                return [*command[:-1], "--model", model, command[-1]]
+        return [*command, "--model", model]
+
+    def command_with_variant(self, command: list[str], variant: str) -> list[str]:
+        return command
+
+    def variant_cli_flag(self) -> str:
+        return ""
+
+    def variant_from_command(self, command: list[str]) -> str:
+        return ""
+
+    def default_worker_tools(self, config: dict[str, Any] | None = None) -> list[str]:
+        return [self.executable(config or {})]
+
+    # -- log directory
+
+    def log_dir_name(self) -> str:
+        return "claude"
+
+    # -- asset installation
+
+    def default_asset_installs(self) -> list[dict[str, Any]]:
+        return [
+            {
+                "source": "work/work/skills/*",
+                "target": ".claude/skills",
+                "kind": "skills",
+                "entries": "directories",
+                "required": False,
+                "clean": True,
+            },
+            {
+                "source": "work/work/skills/versions.json",
+                "target": ".claude/skills",
+                "kind": "skill-manifest",
+                "entries": "files",
+                "required": False,
+                "clean": False,
+            },
+            {
+                "source": "work/skills/*",
+                "target": ".claude/skills",
+                "kind": "skills-flat-fallback",
+                "entries": "directories",
+                "required": False,
+                "clean": False,
+            },
+            {
+                "source": "work/skills/versions.json",
+                "target": ".claude/skills",
+                "kind": "skill-manifest-flat-fallback",
+                "entries": "files",
+                "required": False,
+                "clean": False,
+            },
+            {
+                "source": "work/work/skills/*.md",
+                "target": ".claude/agents",
+                "kind": "agents",
+                "required": False,
+                "clean": True,
+            },
+            {
+                "source": "work/skills/*.md",
+                "target": ".claude/agents",
+                "kind": "agents-flat-fallback",
+                "required": False,
+                "clean": False,
+            },
+        ]
+
+    # -- runtime profile seeding
+
+    def runtime_profile_candidates(
+        self, base_env: dict[str, str], runtime: dict[str, str]
+    ) -> list[tuple[Path | None, Path, str]]:
+        from ..cli import existing_path
+
+        home = Path(runtime["home_dir"])
+        appdata = Path(runtime.get("appdata_dir", home / "AppData" / "Roaming"))
+        localappdata = Path(runtime.get("localappdata_dir", home / "AppData" / "Local"))
+        config_dir = Path(runtime["config_dir"])
+        data_dir = Path(runtime["data_dir"])
+
+        return [
+            (existing_path(base_env.get("USERPROFILE"), ".claude"), home / ".claude", "USERPROFILE/.claude"),
+            (existing_path(base_env.get("HOME"), ".claude"), home / ".claude", "HOME/.claude"),
+            (existing_path(base_env.get("USERPROFILE"), ".config", "claude"), config_dir / "claude", "USERPROFILE/.config/claude"),
+            (existing_path(base_env.get("USERPROFILE"), ".local", "share", "claude"), data_dir / "claude", "USERPROFILE/.local/share/claude"),
+            (existing_path(base_env.get("USERPROFILE"), ".cache", "claude"), Path(runtime["cache_dir"]) / "claude", "USERPROFILE/.cache/claude"),
+            (existing_path(base_env.get("APPDATA"), "Claude"), appdata / "Claude", "APPDATA/Claude"),
+            (existing_path(base_env.get("APPDATA"), "claude"), appdata / "claude", "APPDATA/claude"),
+            (existing_path(base_env.get("LOCALAPPDATA"), "Claude"), localappdata / "Claude", "LOCALAPPDATA/Claude"),
+            (existing_path(base_env.get("LOCALAPPDATA"), "claude"), localappdata / "claude", "LOCALAPPDATA/claude"),
+            (existing_path(base_env.get("XDG_CONFIG_HOME"), "claude"), config_dir / "claude", "XDG_CONFIG_HOME/claude"),
+            (existing_path(base_env.get("XDG_DATA_HOME"), "claude"), data_dir / "claude", "XDG_DATA_HOME/claude"),
+        ]
+
+    # -- provider config
+
+    def write_agent_config(self, manifest: dict[str, Any]) -> str | None:
+        runtime = manifest.get("runtime") or {}
+        home = runtime.get("home_dir")
+        config_dir = Path(home) / ".claude" if home else None
+        if config_dir:
+            config_dir.mkdir(parents=True, exist_ok=True)
+        settings_path = config_dir / "settings.json" if config_dir else None
+
+        from ..cli import agent_inspector_settings, execution_agent_provider
+
+        execution = manifest.setdefault("execution", {})
+        if not isinstance(execution, dict):
+            return str(config_dir) if config_dir else None
+
+        provider = execution_agent_provider(execution, self)
+        inspector = agent_inspector_settings(execution)
+        base_url = ""
+        api_key = ""
+        if isinstance(provider, dict):
+            base_url = str(provider.get("baseURL") or provider.get("base_url") or "").strip()
+            api_key = str(provider.get("apiKey") or provider.get("api_key") or "").strip()
+        if not base_url and inspector.get("enabled"):
+            base_url = str(inspector.get("proxy_url") or "").strip()
+        if base_url:
+            env = execution.setdefault("env", {})
+            if not isinstance(env, dict):
+                raise RuntimeError("execution.env must be a JSON object")
+            env.setdefault("ANTHROPIC_BASE_URL", base_url.rstrip("/"))
+            env.setdefault("ANTHROPIC_API_KEY", api_key or "agent-inspector")
+            env.setdefault("CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", "1")
+            env.setdefault("DISABLE_TELEMETRY", "1")
+            active_model = _claude_active_model(execution)
+            if active_model:
+                env.setdefault("ANTHROPIC_MODEL", active_model)
+                env.setdefault("CLAUDE_CODE_SUBAGENT_MODEL", active_model)
+                for key in _CLAUDE_ALIAS_MODEL_ENV_KEYS:
+                    env.setdefault(key, active_model)
+                if settings_path:
+                    _write_claude_settings_model_pin(settings_path, active_model, env)
+        return str(config_dir) if config_dir else None
+
+    # -- log parsing: sessions
+
+    def extract_sessions_from_text(
+        self, text: str, *, inspector_base_url: str = "http://127.0.0.1:25947"
+    ) -> dict[str, Any]:
+        from ..cli import OPENCODE_SESSION_SCHEMA, plain_log_text, utc_now_seconds_text
+
+        records: dict[str, dict[str, Any]] = {}
+        order: list[str] = []
+        primary_session_id: str | None = None
+
+        def remember(session_id: str) -> dict[str, Any]:
+            if session_id not in records:
+                order.append(session_id)
+                records[session_id] = {
+                    "session_id": session_id,
+                    "parent_id": None,
+                    "agent": None,
+                    "title": None,
+                    "provider": "anthropic",
+                    "model": None,
+                    "mode": None,
+                    "first_seen": None,
+                    "last_seen": None,
+                    "step_count": 0,
+                    "tool_call_count": 0,
+                    "text_part_count": 0,
+                    "error_count": 0,
+                    "tokens": {"input": 0, "output": 0, "reasoning": 0, "cache_read": 0, "cache_write": 0},
+                }
+            return records[session_id]
+
+        for raw_line in text.splitlines():
+            line = plain_log_text(raw_line).strip()
+            if not line.startswith("{"):
+                continue
+            try:
+                payload = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if not isinstance(payload, dict):
+                continue
+            session_id = normalize_agent_value(
+                str(payload.get("session_id") or payload.get("sessionID") or "")
+            )
+            if not session_id:
+                continue
+            record = remember(session_id)
+            timestamp = normalize_agent_value(str(payload.get("timestamp") or ""))
+            _update_seen(record, timestamp)
+            event_type = str(payload.get("type") or "")
+            if event_type == "result":
+                record["step_count"] = int(record.get("step_count", 0)) + 1
+            elif event_type == "error":
+                record["error_count"] = int(record.get("error_count", 0)) + 1
+            message = payload.get("message")
+            if isinstance(message, dict):
+                model = normalize_agent_value(str(message.get("model") or ""))
+                if model:
+                    record["model"] = model
+                _apply_claude_usage(record, message.get("usage"))
+                content = message.get("content")
+                if isinstance(content, list):
+                    for block in content:
+                        if not isinstance(block, dict):
+                            continue
+                        block_type = str(block.get("type") or "")
+                        if block_type == "tool_use":
+                            record["tool_call_count"] = int(record.get("tool_call_count", 0)) + 1
+                        elif block_type == "text":
+                            record["text_part_count"] = int(record.get("text_part_count", 0)) + 1
+            model = normalize_agent_value(str(payload.get("model") or ""))
+            if model:
+                record["model"] = model
+            if primary_session_id is None:
+                primary_session_id = session_id
+
+        sessions = [records[session_id] for session_id in order]
+        base = inspector_base_url.rstrip("/") or "http://127.0.0.1:25947"
+        for record in sessions:
+            record["inspector_url"] = f"{base}/?sessionId={urllib.parse.quote(str(record['session_id']))}"
+        if not primary_session_id and sessions:
+            primary_session_id = str(sessions[0]["session_id"])
+        return {
+            "schema": OPENCODE_SESSION_SCHEMA,
+            "generated_at": utc_now_seconds_text(),
+            "inspector_base_url": base,
+            "inspector_sessions_api": f"{base}/api/sessions",
+            "primary_session_id": primary_session_id,
+            "session_count": len(sessions),
+            "subagent_session_count": sum(1 for item in sessions if item.get("parent_id")),
+            "sessions": sessions,
+        }
+
+    def render_sessions_md(self, summary: dict[str, Any]) -> str:
+        text = super().render_sessions_md(summary)
+        return text.replace("# Claude Sessions", "# Claude Code Sessions", 1)
+
+    # -- log parsing: model
+
+    def model_from_log(self, run_dir: Path) -> str:
+        log_path = run_dir / self.log_dir_name() / "opencode.log"
+        if not log_path.exists():
+            return ""
+        from ..cli import read_trace_text
+
+        try:
+            text = read_trace_text(log_path)
+        except OSError:
+            return ""
+        command_match = re.search(r"^\$\s+.+?\s--model\s+(\S+)", text, re.MULTILINE)
+        if command_match:
+            return command_match.group(1)
+        for line in text.splitlines():
+            line = line.strip()
+            if not line.startswith("{"):
+                continue
+            try:
+                payload = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if not isinstance(payload, dict):
+                continue
+            model = normalize_agent_value(str(payload.get("model") or ""))
+            message = payload.get("message")
+            if not model and isinstance(message, dict):
+                model = normalize_agent_value(str(message.get("model") or ""))
+            if model:
+                return model
+        return ""
+
+    # -- log parsing: activity
+
+    def extract_activity_from_text(self, text: str, *, tail_lines: int = 500) -> dict[str, Any]:
+        from ..cli import plain_log_text
+
+        lines = text.splitlines()[-tail_lines:]
+        sessions: set[str] = set()
+        latest: dict[str, Any] = {}
+        event_count = 0
+        touched_files: list[str] = []
+
+        for raw_line in lines:
+            line = plain_log_text(raw_line).strip()
+            if not line.startswith("{"):
+                continue
+            try:
+                payload = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if not isinstance(payload, dict):
+                continue
+            session_id = normalize_agent_value(str(payload.get("session_id") or payload.get("sessionID") or ""))
+            if not session_id:
+                continue
+            event_count += 1
+            sessions.add(session_id)
+            message = payload.get("message")
+            model = normalize_agent_value(str(payload.get("model") or ""))
+            if not model and isinstance(message, dict):
+                model = normalize_agent_value(str(message.get("model") or ""))
+            latest = {
+                "latest_session_id": session_id,
+                "latest_mode": None,
+                "latest_agent": None,
+                "latest_model": model,
+                "latest_message": normalize_agent_value(str(payload.get("type") or "")),
+                "latest_timestamp": normalize_agent_value(str(payload.get("timestamp") or "")),
+            }
+            if isinstance(message, dict) and isinstance(message.get("content"), list):
+                for block in message["content"]:
+                    if isinstance(block, dict) and block.get("type") == "tool_use":
+                        name = str(block.get("name") or "")
+                        if name in {"Edit", "Write", "MultiEdit"}:
+                            tool_input = block.get("input")
+                            if isinstance(tool_input, dict) and tool_input.get("file_path"):
+                                touched_files.append(str(tool_input["file_path"]))
+
+        latest.update(
+            {
+                "event_count": event_count,
+                "session_count": len(sessions),
+                "subagent_session_count": 0,
+                "has_subagent_activity": False,
+                "touched_file_count": len(touched_files),
+                "latest_touched_file": touched_files[-1] if touched_files else "",
+            }
+        )
+        return latest
+
+
+_CLAUDE_VALUE_FLAGS = {
+    "--add-dir",
+    "--agent",
+    "--agents",
+    "--allowedTools",
+    "--allowed-tools",
+    "--append-system-prompt",
+    "--debug-file",
+    "--disallowedTools",
+    "--disallowed-tools",
+    "--effort",
+    "--fallback-model",
+    "--file",
+    "--input-format",
+    "--json-schema",
+    "--max-budget-usd",
+    "--mcp-config",
+    "--model",
+    "-m",
+    "--name",
+    "-n",
+    "--output-format",
+    "--permission-mode",
+    "--plugin-dir",
+    "--plugin-url",
+    "--resume",
+    "-r",
+    "--session-id",
+    "--setting-sources",
+    "--settings",
+    "--system-prompt",
+    "--tools",
+}
+
+
+def _is_claude_executable(value: str) -> bool:
+    name = Path(value).name.lower()
+    return name in {"claude", "claude.exe", "claude.cmd", "claude.ps1"}
+
+
+def _claude_model_name(model: str) -> str:
+    text = str(model or "").strip()
+    prefix = "agent-inspector/"
+    if text.startswith(prefix):
+        return text[len(prefix) :]
+    return text
+
+
+_CLAUDE_ALIAS_MODEL_ENV_KEYS = (
+    "ANTHROPIC_DEFAULT_FABLE_MODEL",
+    "ANTHROPIC_DEFAULT_OPUS_MODEL",
+    "ANTHROPIC_DEFAULT_SONNET_MODEL",
+    "ANTHROPIC_DEFAULT_HAIKU_MODEL",
+)
+
+
+def _claude_active_model(execution: dict[str, Any]) -> str:
+    for key in ("actual_model", "model"):
+        value = _claude_model_name(str(execution.get(key) or "")).strip()
+        if value:
+            return value
+    env = execution.get("env")
+    if isinstance(env, dict):
+        for key in ("ANTHROPIC_MODEL", "CLAUDE_CODE_SUBAGENT_MODEL"):
+            value = _claude_model_name(str(env.get(key) or "")).strip()
+            if value and value != "inherit":
+                return value
+    return ""
+
+
+def _write_claude_settings_model_pin(settings_path: Path, active_model: str, env: dict[str, Any]) -> None:
+    settings: dict[str, Any] = {}
+    if settings_path.exists():
+        try:
+            loaded = json.loads(settings_path.read_text(encoding="utf-8-sig"))
+            if isinstance(loaded, dict):
+                settings = loaded
+        except json.JSONDecodeError:
+            settings = {}
+    settings["model"] = str(env.get("ANTHROPIC_MODEL") or active_model)
+    settings_env = settings.setdefault("env", {})
+    if not isinstance(settings_env, dict):
+        settings_env = {}
+        settings["env"] = settings_env
+    for key in ("ANTHROPIC_MODEL", "CLAUDE_CODE_SUBAGENT_MODEL", *_CLAUDE_ALIAS_MODEL_ENV_KEYS):
+        value = env.get(key)
+        if value:
+            settings_env[key] = str(value)
+    settings_path.parent.mkdir(parents=True, exist_ok=True)
+    settings_path.write_text(json.dumps(settings, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+
+
+def _update_seen(record: dict[str, Any], timestamp: str | None) -> None:
+    if not timestamp:
+        return
+    if record.get("first_seen") is None or str(timestamp) < str(record.get("first_seen")):
+        record["first_seen"] = timestamp
+    if record.get("last_seen") is None or str(timestamp) > str(record.get("last_seen")):
+        record["last_seen"] = timestamp
+
+
+def _apply_claude_usage(record: dict[str, Any], usage: Any) -> None:
+    if not isinstance(usage, dict):
+        return
+    tokens = record.setdefault("tokens", {})
+    mapping = {
+        "input_tokens": "input",
+        "output_tokens": "output",
+        "cache_read_input_tokens": "cache_read",
+        "cache_creation_input_tokens": "cache_write",
+    }
+    for source, target in mapping.items():
+        value = usage.get(source)
+        if isinstance(value, int):
+            tokens[target] = int(tokens.get(target, 0)) + value