eval-harness 0.2.18

package/src/eval/runners/opencode.py

@@ -0,0 +1,554 @@
+from __future__ import annotations
+
+import json
+import os
+import re
+import urllib.parse
+from pathlib import Path
+from typing import Any
+
+from .base import (
+    AgentRunner,
+    DEFAULT_ATTACHED_INSTRUCTION_PROMPT,
+    _command_get_named_arg,
+    _command_set_named_arg,
+    normalize_agent_value,
+    parse_log_key_values,
+)
+
+
+class OpenCodeRunner(AgentRunner):
+    """OpenCode CLI agent runner adapter."""
+
+    @property
+    def name(self) -> str:
+        return "opencode"
+
+    @property
+    def binary(self) -> str:
+        return "opencode"
+
+    @property
+    def config_key(self) -> str:
+        return "opencode"
+
+    # -- command building
+
+    def default_command(self, config: dict[str, Any]) -> list[str]:
+        return [
+            "opencode",
+            "run",
+            DEFAULT_ATTACHED_INSTRUCTION_PROMPT,
+            *self.format_flags(),
+            "--dangerously-skip-permissions",
+            "--file={instruction}",
+        ]
+
+    def command_with_variant(self, command: list[str], variant: str) -> list[str]:
+        if not variant:
+            return command
+        return _command_set_named_arg(command, "--variant", variant)
+
+    def variant_cli_flag(self) -> str:
+        return "--variant"
+
+    def variant_from_command(self, command: list[str]) -> str:
+        return _command_get_named_arg(command, "--variant")
+
+    # -- log directory
+
+    def log_dir_name(self) -> str:
+        return "opencode"
+
+    # -- asset installation
+
+    def default_asset_installs(self) -> list[dict[str, Any]]:
+        return [
+            {
+                "source": "work/work/skills/*",
+                "target": ".opencode/skills",
+                "kind": "skills",
+                "entries": "directories",
+                "required": False,
+                "clean": True,
+            },
+            {
+                "source": "work/work/skills/versions.json",
+                "target": ".opencode/skills",
+                "kind": "skill-manifest",
+                "entries": "files",
+                "required": False,
+                "clean": False,
+            },
+            {
+                "source": "work/skills/*",
+                "target": ".opencode/skills",
+                "kind": "skills-flat-fallback",
+                "entries": "directories",
+                "required": False,
+                "clean": False,
+            },
+            {
+                "source": "work/skills/versions.json",
+                "target": ".opencode/skills",
+                "kind": "skill-manifest-flat-fallback",
+                "entries": "files",
+                "required": False,
+                "clean": False,
+            },
+            {
+                "source": "work/work/skills/*.md",
+                "target": ".opencode/agents",
+                "kind": "agents",
+                "required": False,
+                "clean": True,
+            },
+            {
+                "source": "work/skills/*.md",
+                "target": ".opencode/agents",
+                "kind": "agents-flat-fallback",
+                "required": False,
+                "clean": False,
+            },
+        ]
+
+    # -- runtime profile seeding
+
+    def runtime_profile_candidates(
+        self, base_env: dict[str, str], runtime: dict[str, str]
+    ) -> list[tuple[Path | None, Path, str]]:
+        from ..cli import existing_path
+
+        # Dest paths use the isolated runtime dirs; source paths use the real env
+        home = Path(runtime["home_dir"])
+        appdata = Path(runtime.get("appdata_dir", home / "AppData" / "Roaming"))
+        localappdata = Path(runtime.get("localappdata_dir", home / "AppData" / "Local"))
+        config_dir = Path(runtime["config_dir"])
+        data_dir = Path(runtime["data_dir"])
+
+        return [
+            (existing_path(base_env.get("USERPROFILE"), ".config", "opencode"), config_dir / "opencode", "USERPROFILE/.config/opencode"),
+            (existing_path(base_env.get("USERPROFILE"), ".local", "share", "opencode"), data_dir / "opencode", "USERPROFILE/.local/share/opencode"),
+            (existing_path(base_env.get("USERPROFILE"), ".cache", "opencode"), Path(runtime["cache_dir"]) / "opencode", "USERPROFILE/.cache/opencode"),
+            (existing_path(base_env.get("USERPROFILE"), ".opencode"), home / ".opencode", "USERPROFILE/.opencode"),
+            (existing_path(base_env.get("APPDATA"), "opencode"), appdata / "opencode", "APPDATA/opencode"),
+            (existing_path(base_env.get("LOCALAPPDATA"), "opencode"), localappdata / "opencode", "LOCALAPPDATA/opencode"),
+            (existing_path(base_env.get("XDG_CONFIG_HOME"), "opencode"), config_dir / "opencode", "XDG_CONFIG_HOME/opencode"),
+            (existing_path(base_env.get("XDG_DATA_HOME"), "opencode"), data_dir / "opencode", "XDG_DATA_HOME/opencode"),
+        ]
+
+    # -- provider config
+
+    def write_agent_config(self, manifest: dict[str, Any]) -> str | None:
+        runtime = manifest.get("runtime") or {}
+        config_dir = Path(runtime["config_dir"]) / "opencode"
+        config_dir.mkdir(parents=True, exist_ok=True)
+        config_path = config_dir / "opencode.jsonc"
+
+        from ..cli import execution_agent_provider, read_json, write_json  # local to avoid circular
+
+        data: dict[str, Any] = {}
+        if config_path.exists():
+            try:
+                data = read_json(config_path)
+            except json.JSONDecodeError:
+                data = {}
+        data.setdefault("$schema", "https://opencode.ai/config.json")
+        providers = data.setdefault("provider", {})
+        if not isinstance(providers, dict):
+            raise RuntimeError("opencode provider config must be a JSON object")
+
+        execution = manifest.get("execution", {})
+        active_model = _opencode_active_model(execution)
+        if active_model:
+            data["model"] = active_model
+        provider = execution_agent_provider(execution, self)
+        if not isinstance(provider, dict):
+            return str(config_path)
+
+        provider_id = str(provider.get("id", "agent-inspector"))
+        models = provider.get("models") or {}
+        if not isinstance(models, dict):
+            raise RuntimeError("execution.opencode_provider.models must be a JSON object")
+        providers[provider_id] = {
+            "npm": str(provider.get("npm", "@ai-sdk/openai-compatible")),
+            "name": str(provider.get("name", "Agent Inspector")),
+            "options": {
+                "baseURL": str(provider.get("baseURL", "http://127.0.0.1:25947/proxy")),
+                "apiKey": str(provider.get("apiKey", "")),
+            },
+            "models": models,
+        }
+        write_json(config_path, data)
+        return str(config_path)
+
+    # -- log parsing: sessions
+
+    def extract_sessions_from_text(
+        self, text: str, *, inspector_base_url: str = "http://127.0.0.1:25947"
+    ) -> dict[str, Any]:
+        from ..cli import (
+            OPENCODE_SESSION_SCHEMA,
+            dt,
+            plain_log_text,
+            utc_now_seconds_text,
+        )
+
+        records: dict[str, dict[str, Any]] = {}
+        order: list[str] = []
+        primary_session_id: str | None = None
+
+        def remember(session_id: str) -> dict[str, Any]:
+            if session_id not in records:
+                order.append(session_id)
+            return _ensure_session_record(records, session_id)
+
+        for raw_line in text.splitlines():
+            line = plain_log_text(raw_line).strip()
+            if not line:
+                continue
+            if line.startswith("{"):
+                try:
+                    payload = json.loads(line)
+                except json.JSONDecodeError:
+                    continue
+                if not isinstance(payload, dict):
+                    continue
+                session_id = normalize_agent_value(str(payload.get("sessionID") or ""))
+                part = payload.get("part")
+                if not session_id and isinstance(part, dict):
+                    session_id = normalize_agent_value(str(part.get("sessionID") or ""))
+                if not session_id:
+                    continue
+                record = remember(session_id)
+                _update_session_seen(record, _session_timestamp_from_json(payload))
+                event_type = str(payload.get("type") or "")
+                if event_type == "tool_use":
+                    record["tool_call_count"] = int(record.get("tool_call_count", 0)) + 1
+                elif event_type == "text":
+                    record["text_part_count"] = int(record.get("text_part_count", 0)) + 1
+                elif event_type == "error":
+                    record["error_count"] = int(record.get("error_count", 0)) + 1
+                if isinstance(part, dict):
+                    if part.get("type") == "step-finish":
+                        record["step_count"] = int(record.get("step_count", 0)) + 1
+                        tokens = part.get("tokens")
+                        if isinstance(tokens, dict):
+                            _apply_step_tokens(record, tokens)
+                    if not record.get("agent"):
+                        record["agent"] = normalize_agent_value(str(part.get("agent") or ""))
+                if primary_session_id is None:
+                    primary_session_id = session_id
+                continue
+
+            values = parse_log_key_values(line)
+            timestamp = normalize_agent_value(values.get("timestamp"))
+            created_id = normalize_agent_value(values.get("id")) if "message=created" in line else None
+            stream_session_id = normalize_agent_value(values.get("session.id"))
+            session_id = created_id or stream_session_id
+            if not session_id:
+                continue
+            record = remember(session_id)
+            _update_session_seen(record, timestamp)
+            if created_id:
+                parent_id = normalize_agent_value(values.get("parentID"))
+                record["parent_id"] = parent_id
+                record["title"] = normalize_agent_value(values.get("title")) or record.get("title")
+                record["agent"] = normalize_agent_value(values.get("agent")) or record.get("agent")
+                model = normalize_agent_value(values.get("model"))
+                if model:
+                    record["model"] = model
+                if parent_id is None and primary_session_id is None:
+                    primary_session_id = session_id
+            if stream_session_id:
+                record["provider"] = normalize_agent_value(values.get("providerID")) or normalize_agent_value(values.get("llm.provider")) or record.get("provider")
+                record["model"] = normalize_agent_value(values.get("modelID")) or normalize_agent_value(values.get("llm.model")) or record.get("model")
+                record["agent"] = normalize_agent_value(values.get("agent")) or record.get("agent")
+                record["mode"] = normalize_agent_value(values.get("mode")) or record.get("mode")
+                if primary_session_id is None:
+                    primary_session_id = session_id
+
+        sessions = [records[session_id] for session_id in order]
+        base = inspector_base_url.rstrip("/") or "http://127.0.0.1:25947"
+        for record in sessions:
+            record["inspector_url"] = f"{base}/?sessionId={urllib.parse.quote(str(record['session_id']))}"
+        if not primary_session_id and sessions:
+            primary_session_id = str(sessions[0]["session_id"])
+        return {
+            "schema": OPENCODE_SESSION_SCHEMA,
+            "generated_at": utc_now_seconds_text(),
+            "inspector_base_url": base,
+            "inspector_sessions_api": f"{base}/api/sessions",
+            "primary_session_id": primary_session_id,
+            "session_count": len(sessions),
+            "subagent_session_count": sum(1 for item in sessions if item.get("parent_id")),
+            "sessions": sessions,
+        }
+
+    def render_sessions_md(self, summary: dict[str, Any]) -> str:
+        from ..cli import md_artifact_external_link, md_cell
+
+        lines = [
+            f"# {self.binary.title()} Sessions",
+            "",
+            f"- Generated at: {summary.get('generated_at')}",
+            f"- Inspector: {summary.get('inspector_base_url')}",
+            f"- Sessions API: {summary.get('inspector_sessions_api')}",
+            f"- Primary session: `{summary.get('primary_session_id') or ''}`",
+            f"- Session count: {summary.get('session_count', 0)}",
+            f"- Subagent session count: {summary.get('subagent_session_count', 0)}",
+            "",
+            "| session | parent | agent | mode | model | steps | tools | errors | inspector |",
+            "| --- | --- | --- | --- | --- | ---: | ---: | ---: | --- |",
+        ]
+        for item in summary.get("sessions", []) or []:
+            lines.append(
+                "| "
+                + " | ".join(
+                    [
+                        md_cell(item.get("session_id")),
+                        md_cell(item.get("parent_id")),
+                        md_cell(item.get("agent")),
+                        md_cell(item.get("mode")),
+                        md_cell(item.get("model")),
+                        md_cell(item.get("step_count")),
+                        md_cell(item.get("tool_call_count")),
+                        md_cell(item.get("error_count")),
+                        md_artifact_external_link(str(item.get("inspector_url") or ""), "open"),
+                    ]
+                )
+                + " |"
+            )
+        if not summary.get("sessions"):
+            lines.append("| none |  |  |  |  | 0 | 0 | 0 |  |")
+        if summary.get("inspector_backfill_status") and summary.get("inspector_backfill_status") != "SKIPPED":
+            lines.extend(
+                [
+                    "",
+                    "## Inspector Log Backfill",
+                    "",
+                    f"- Status: `{summary.get('inspector_backfill_status') or ''}`",
+                    f"- Query mode: `{summary.get('inspector_log_query_mode') or ''}`",
+                    f"- Logs URL: {md_artifact_external_link(str(summary.get('inspector_logs_url') or ''), 'open')}",
+                    f"- Log count: {summary.get('inspector_log_count', 0)}",
+                    f"- Preview count: {summary.get('inspector_log_preview_count', 0)} / {summary.get('inspector_log_preview_limit', 0)}",
+                ]
+            )
+            if summary.get("inspector_logs_artifact"):
+                lines.append(f"- Full log artifact: `{summary.get('inspector_logs_artifact')}`")
+            if summary.get("inspector_backfill_reason"):
+                lines.append(f"- Reason: {summary.get('inspector_backfill_reason')}")
+            inspector_logs = summary.get("inspector_logs") or []
+            if inspector_logs:
+                lines.extend(
+                    [
+                        "",
+                        "| log | session | model | status | path | inspector |",
+                        "| ---: | --- | --- | ---: | --- | --- |",
+                    ]
+                )
+                for item in inspector_logs:
+                    lines.append(
+                        "| "
+                        + " | ".join(
+                            [
+                                md_cell(item.get("id")),
+                                md_cell(item.get("session_id")),
+                                md_cell(item.get("model")),
+                                md_cell(item.get("status")),
+                                md_cell(item.get("path")),
+                                md_artifact_external_link(str(item.get("inspector_url") or ""), "open"),
+                            ]
+                        )
+                        + " |"
+                    )
+        lines.append("")
+        return "\n".join(lines)
+
+    # -- log parsing: model
+
+    def model_from_log(self, run_dir: Path) -> str:
+        log_path = run_dir / self.log_dir_name() / "opencode.log"
+        if not log_path.exists():
+            return ""
+        from ..cli import read_trace_text
+
+        try:
+            text = read_trace_text(log_path)
+        except OSError:
+            return ""
+        command_match = re.search(r"^\$\s+.+?\s--model\s+(\S+)", text, re.MULTILINE)
+        if command_match:
+            return command_match.group(1)
+        stream_match = re.search(r"providerID=(\S+)\s+modelID=(\S+)", text)
+        if stream_match:
+            return f"{stream_match.group(1)}/{stream_match.group(2)}"
+        runtime_match = re.search(r"llm\.provider=(\S+)\s+llm\.model=(\S+)", text)
+        if runtime_match:
+            return f"{runtime_match.group(1)}/{runtime_match.group(2)}"
+        return ""
+
+    # -- log parsing: activity
+
+    def extract_activity_from_text(self, text: str, *, tail_lines: int = 500) -> dict[str, Any]:
+        from ..cli import plain_log_text
+
+        lines = text.splitlines()[-tail_lines:]
+        sessions: set[str] = set()
+        subagent_sessions: set[str] = set()
+        touched_files: list[str] = []
+        latest: dict[str, Any] = {}
+        event_count = 0
+
+        def remember(
+            session_id: str | None,
+            *,
+            mode: str | None = None,
+            agent: str | None = None,
+            model: str | None = None,
+            message: str | None = None,
+            timestamp: str | None = None,
+        ) -> None:
+            nonlocal event_count, latest
+            normalized_session = normalize_agent_value(session_id)
+            if not normalized_session:
+                return
+            event_count += 1
+            sessions.add(normalized_session)
+            normalized_mode = normalize_agent_value(mode)
+            if normalized_mode == "subagent":
+                subagent_sessions.add(normalized_session)
+            latest = {
+                "latest_session_id": normalized_session,
+                "latest_mode": normalized_mode,
+                "latest_agent": normalize_agent_value(agent),
+                "latest_model": normalize_agent_value(model),
+                "latest_message": normalize_agent_value(message),
+                "latest_timestamp": normalize_agent_value(timestamp),
+            }
+
+        for raw_line in lines:
+            line = plain_log_text(raw_line).strip()
+            if not line:
+                continue
+            if line.startswith("{"):
+                try:
+                    payload = json.loads(line)
+                except json.JSONDecodeError:
+                    payload = None
+                if isinstance(payload, dict):
+                    part = payload.get("part") if isinstance(payload.get("part"), dict) else {}
+                    session_id = payload.get("sessionID") or part.get("sessionID")
+                    remember(
+                        str(session_id or ""),
+                        mode=str(part.get("mode") or ""),
+                        agent=str(part.get("agent") or ""),
+                        message=str(payload.get("type") or part.get("type") or ""),
+                        timestamp=str(payload.get("timestamp") or ""),
+                    )
+                    continue
+
+            values = parse_log_key_values(line)
+            session_id = values.get("session.id")
+            if not session_id and "message=created" in line:
+                session_id = values.get("id")
+            if session_id:
+                remember(
+                    session_id,
+                    mode=values.get("mode"),
+                    agent=values.get("agent"),
+                    model=values.get("modelID") or values.get("llm.model") or values.get("model"),
+                    message=values.get("message"),
+                    timestamp=values.get("timestamp"),
+                )
+            touched = values.get("file") if values.get("message") == "touching file" else None
+            if touched:
+                touched_files.append(touched)
+
+        latest.update(
+            {
+                "event_count": event_count,
+                "session_count": len(sessions),
+                "session_ids": sorted(sessions),
+                "subagent_session_count": len(subagent_sessions),
+                "subagent_session_ids": sorted(subagent_sessions),
+                "has_subagent_activity": bool(subagent_sessions) or latest.get("latest_mode") == "subagent",
+                "touched_file_count": len(touched_files),
+                "latest_touched_file": touched_files[-1] if touched_files else "",
+            }
+        )
+        return latest
+
+
+# -- shared session helpers
+
+def _opencode_active_model(execution: dict[str, Any]) -> str:
+    for key in ("actual_model", "model"):
+        value = normalize_agent_value(str(execution.get(key) or ""))
+        if value and value != "inherit":
+            return value
+    return ""
+
+
+def _ensure_session_record(
+    records: dict[str, dict[str, Any]], session_id: str
+) -> dict[str, Any]:
+    record = records.get(session_id)
+    if record is None:
+        record = {
+            "session_id": session_id,
+            "parent_id": None,
+            "agent": None,
+            "title": None,
+            "provider": None,
+            "model": None,
+            "mode": None,
+            "first_seen": None,
+            "last_seen": None,
+            "step_count": 0,
+            "tool_call_count": 0,
+            "text_part_count": 0,
+            "error_count": 0,
+            "tokens": {"input": 0, "output": 0, "reasoning": 0, "cache_read": 0, "cache_write": 0},
+        }
+        records[session_id] = record
+    return record
+
+
+def _update_session_seen(record: dict[str, Any], timestamp: str | None) -> None:
+    if not timestamp:
+        return
+    if record.get("first_seen") is None or str(timestamp) < str(record.get("first_seen")):
+        record["first_seen"] = timestamp
+    if record.get("last_seen") is None or str(timestamp) > str(record.get("last_seen")):
+        record["last_seen"] = timestamp
+
+
+def _session_timestamp_from_json(payload: dict[str, Any]) -> str | None:
+    import datetime as dt
+
+    value = payload.get("timestamp")
+    if isinstance(value, (int, float)):
+        try:
+            return dt.datetime.fromtimestamp(value / 1000, tz=dt.timezone.utc).isoformat()
+        except (OverflowError, OSError, ValueError):
+            return str(value)
+    if value is None:
+        return None
+    return str(value)
+
+
+def _apply_step_tokens(record: dict[str, Any], tokens: dict[str, Any]) -> None:
+    bucket = record.setdefault("tokens", {})
+    for key in ("input", "output", "reasoning"):
+        value = tokens.get(key)
+        if isinstance(value, int):
+            bucket[key] = int(bucket.get(key, 0)) + value
+    cache = tokens.get("cache")
+    if isinstance(cache, dict):
+        read = cache.get("read")
+        write = cache.get("write")
+        if isinstance(read, int):
+            bucket["cache_read"] = int(bucket.get("cache_read", 0)) + read
+        if isinstance(write, int):
+            bucket["cache_write"] = int(bucket.get("cache_write", 0)) + write