eval-harness 0.2.18

package/src/eval/runners/mimocode.py

@@ -0,0 +1,676 @@
+from __future__ import annotations
+
+import json
+import os
+import re
+import urllib.parse
+from pathlib import Path
+from typing import Any
+
+from .base import (
+    AgentRunner,
+    DEFAULT_ATTACHED_INSTRUCTION_PROMPT,
+    _command_get_named_arg,
+    _command_set_named_arg,
+    normalize_agent_value,
+    parse_log_key_values,
+)
+
+
+class MiMoCodeRunner(AgentRunner):
+    """MiMo Code CLI agent runner adapter."""
+
+    harness_agent_name = "harness-build"
+
+    @property
+    def name(self) -> str:
+        return "mimocode"
+
+    @property
+    def binary(self) -> str:
+        return "mimo"
+
+    @property
+    def config_key(self) -> str:
+        return "mimocode"
+
+    # -- command building
+
+    def default_command(self, config: dict[str, Any]) -> list[str]:
+        return [
+            "mimo",
+            "run",
+            *self.format_flags(),
+            "--dangerously-skip-permissions",
+            "--file={instruction}",
+            "--dir={workspace}",
+            "--agent",
+            self.harness_agent_name,
+            DEFAULT_ATTACHED_INSTRUCTION_PROMPT,
+        ]
+
+    def command_with_model(self, command: list[str], model: str) -> list[str]:
+        """Inject --model before MiMo's positional message argument."""
+        return _mimocode_set_option_before_message(command, "--model", model)
+
+    def command_with_variant(self, command: list[str], variant: str) -> list[str]:
+        """MiMo Code uses --agent instead of --variant."""
+        if not variant:
+            return command
+        if self._looks_like_reasoning_variant(variant):
+            return command
+        return _command_set_named_arg(command, "--agent", variant)
+
+    def variant_cli_flag(self) -> str:
+        return "--agent"
+
+    def variant_from_command(self, command: list[str]) -> str:
+        return _command_get_named_arg(command, "--agent")
+
+    # -- log directory
+
+    def log_dir_name(self) -> str:
+        return "mimocode"
+
+    # -- asset installation
+
+    def default_asset_installs(self) -> list[dict[str, Any]]:
+        return [
+            {
+                "source": "work/work/skills/*",
+                "target": ".mimocode/skills",
+                "kind": "skills",
+                "entries": "directories",
+                "required": False,
+                "clean": True,
+            },
+            {
+                "source": "work/work/skills/versions.json",
+                "target": ".mimocode/skills",
+                "kind": "skill-manifest",
+                "entries": "files",
+                "required": False,
+                "clean": False,
+            },
+            {
+                "source": "work/skills/*",
+                "target": ".mimocode/skills",
+                "kind": "skills-flat-fallback",
+                "entries": "directories",
+                "required": False,
+                "clean": False,
+            },
+            {
+                "source": "work/skills/versions.json",
+                "target": ".mimocode/skills",
+                "kind": "skill-manifest-flat-fallback",
+                "entries": "files",
+                "required": False,
+                "clean": False,
+            },
+            {
+                "source": "work/work/skills/*.md",
+                "target": ".mimocode/agents",
+                "kind": "agents",
+                "required": False,
+                "clean": True,
+            },
+            {
+                "source": "work/skills/*.md",
+                "target": ".mimocode/agents",
+                "kind": "agents-flat-fallback",
+                "required": False,
+                "clean": False,
+            },
+        ]
+
+    # -- runtime profile seeding
+
+    def runtime_profile_candidates(
+        self, base_env: dict[str, str], runtime: dict[str, str]
+    ) -> list[tuple[Path | None, Path, str]]:
+        from ..cli import existing_path
+
+        home = Path(runtime["home_dir"])
+        appdata = Path(runtime.get("appdata_dir", home / "AppData" / "Roaming"))
+        localappdata = Path(runtime.get("localappdata_dir", home / "AppData" / "Local"))
+        config_dir = Path(runtime["config_dir"])
+        data_dir = Path(runtime["data_dir"])
+
+        return [
+            (existing_path(base_env.get("USERPROFILE"), ".config", "mimocode"), config_dir / "mimocode", "USERPROFILE/.config/mimocode"),
+            (existing_path(base_env.get("USERPROFILE"), ".local", "share", "mimocode"), data_dir / "mimocode", "USERPROFILE/.local/share/mimocode"),
+            (existing_path(base_env.get("USERPROFILE"), ".cache", "mimocode"), Path(runtime["cache_dir"]) / "mimocode", "USERPROFILE/.cache/mimocode"),
+            (existing_path(base_env.get("USERPROFILE"), ".mimocode"), home / ".mimocode", "USERPROFILE/.mimocode"),
+            (existing_path(base_env.get("APPDATA"), "mimocode"), appdata / "mimocode", "APPDATA/mimocode"),
+            (existing_path(base_env.get("LOCALAPPDATA"), "mimocode"), localappdata / "mimocode", "LOCALAPPDATA/mimocode"),
+            (existing_path(base_env.get("XDG_CONFIG_HOME"), "mimocode"), config_dir / "mimocode", "XDG_CONFIG_HOME/mimocode"),
+            (existing_path(base_env.get("XDG_DATA_HOME"), "mimocode"), data_dir / "mimocode", "XDG_DATA_HOME/mimocode"),
+        ]
+
+    # -- provider config
+
+    def write_agent_config(self, manifest: dict[str, Any]) -> str | None:
+        runtime = manifest.get("runtime") or {}
+        config_dir = Path(runtime["config_dir"]) / "mimocode"
+        config_dir.mkdir(parents=True, exist_ok=True)
+        config_path = config_dir / "mimocode.jsonc"
+
+        from ..cli import execution_agent_provider, read_json, write_json
+
+        data: dict[str, Any] = {}
+        if config_path.exists():
+            try:
+                data = read_json(config_path)
+            except json.JSONDecodeError:
+                data = {}
+        data.setdefault("$schema", "https://mimo.xiaomi.com/config.json")
+        execution = manifest.get("execution", {})
+        active_model = _mimocode_active_model(execution)
+        if active_model:
+            data["model"] = active_model
+
+        agents = data.setdefault("agent", {})
+        if not isinstance(agents, dict):
+            raise RuntimeError("mimocode agent config must be a JSON object")
+        agents[self.harness_agent_name] = self._harness_agent_config(active_model)
+        if active_model:
+            build_agent = agents.get("build")
+            if not isinstance(build_agent, dict):
+                build_agent = {}
+            build_agent["model"] = active_model
+            agents["build"] = build_agent
+        providers = data.setdefault("provider", {})
+        if not isinstance(providers, dict):
+            raise RuntimeError("mimocode provider config must be a JSON object")
+
+        provider = execution_agent_provider(execution, self)
+        if not isinstance(provider, dict):
+            return str(config_path)
+
+        provider_id = str(provider.get("id", "agent-inspector"))
+        models = provider.get("models") or {}
+        if not isinstance(models, dict):
+            raise RuntimeError("execution.opencode_provider.models must be a JSON object")
+        providers[provider_id] = {
+            "npm": str(provider.get("npm", "@ai-sdk/openai-compatible")),
+            "name": str(provider.get("name", "Agent Inspector")),
+            "options": {
+                "baseURL": str(provider.get("baseURL", "http://127.0.0.1:25947/proxy")),
+                "apiKey": str(provider.get("apiKey", "")),
+            },
+            "models": models,
+        }
+        write_json(config_path, data)
+        return str(config_path)
+
+    @staticmethod
+    def _looks_like_reasoning_variant(value: str) -> bool:
+        return value.strip().lower() in {
+            "minimal",
+            "low",
+            "medium",
+            "high",
+            "max",
+            "maximum",
+        }
+
+    def _harness_agent_config(self, active_model: str = "") -> dict[str, Any]:
+        config: dict[str, Any] = {
+            "description": (
+                "Eval Harness non-interactive build agent. It mirrors MiMo Code's "
+                "build mode, but pre-approves external_directory access so Jenkins runs "
+                "do not block on permission prompts."
+            ),
+            "mode": "primary",
+            "permission": {
+                "read": "allow",
+                "edit": "allow",
+                "glob": "allow",
+                "grep": "allow",
+                "list": "allow",
+                "bash": "allow",
+                "task": "allow",
+                "external_directory": "allow",
+                "todowrite": "allow",
+                "question": "deny",
+                "webfetch": "allow",
+                "websearch": "allow",
+                "lsp": "allow",
+                "doom_loop": "allow",
+                "skill": "allow",
+            },
+            "tools": {
+                "invalid": True,
+                "question": True,
+                "bash": True,
+                "read": True,
+                "glob": True,
+                "grep": True,
+                "edit": True,
+                "write": True,
+                "notebook_edit": True,
+                "actor": True,
+                "webfetch": True,
+                "skill": True,
+                "change_directory": True,
+                "plan_exit": True,
+                "plan_enter": True,
+                "memory": True,
+                "history": True,
+                "task": True,
+                "workflow": True,
+            },
+        }
+        if active_model:
+            config["model"] = active_model
+        return config
+
+    # -- log parsing: sessions
+
+    def extract_sessions_from_text(
+        self, text: str, *, inspector_base_url: str = "http://127.0.0.1:25947"
+    ) -> dict[str, Any]:
+        from ..cli import (
+            OPENCODE_SESSION_SCHEMA,
+            dt,
+            plain_log_text,
+            utc_now_seconds_text,
+        )
+
+        records: dict[str, dict[str, Any]] = {}
+        order: list[str] = []
+        primary_session_id: str | None = None
+
+        def remember(session_id: str) -> dict[str, Any]:
+            if session_id not in records:
+                order.append(session_id)
+            return _ensure_session_record(records, session_id)
+
+        for raw_line in text.splitlines():
+            line = plain_log_text(raw_line).strip()
+            if not line:
+                continue
+            if line.startswith("{"):
+                try:
+                    payload = json.loads(line)
+                except json.JSONDecodeError:
+                    continue
+                if not isinstance(payload, dict):
+                    continue
+                # MiMo Code JSON stream: sessionID / part.sessionID
+                session_id = normalize_agent_value(str(payload.get("sessionID") or ""))
+                part = payload.get("part")
+                if not session_id and isinstance(part, dict):
+                    session_id = normalize_agent_value(str(part.get("sessionID") or ""))
+                # Also try MiMo-specific: turnID / session.id
+                if not session_id:
+                    session_id = normalize_agent_value(str(payload.get("turnID") or ""))
+                if not session_id and isinstance(part, dict):
+                    session_id = normalize_agent_value(str(part.get("turnID") or ""))
+                if not session_id:
+                    continue
+                record = remember(session_id)
+                _update_session_seen(record, _session_timestamp_from_json(payload))
+                event_type = str(payload.get("type") or "")
+                if event_type in ("tool_use", "tool-result"):
+                    record["tool_call_count"] = int(record.get("tool_call_count", 0)) + 1
+                elif event_type in ("text", "content_block_start", "content_block_delta"):
+                    record["text_part_count"] = int(record.get("text_part_count", 0)) + 1
+                elif event_type == "error":
+                    record["error_count"] = int(record.get("error_count", 0)) + 1
+                if isinstance(part, dict):
+                    if part.get("type") in ("step-finish", "turn-finish", "done"):
+                        record["step_count"] = int(record.get("step_count", 0)) + 1
+                        tokens = part.get("tokens") or part.get("usage")
+                        if isinstance(tokens, dict):
+                            _apply_step_tokens(record, tokens)
+                    if not record.get("agent"):
+                        record["agent"] = normalize_agent_value(
+                            str(part.get("agent") or part.get("name") or "")
+                        )
+                if primary_session_id is None:
+                    primary_session_id = session_id
+                continue
+
+            # key=value log lines (shared format)
+            values = parse_log_key_values(line)
+            timestamp = normalize_agent_value(values.get("timestamp"))
+            created_id = normalize_agent_value(values.get("id")) if "message=created" in line else None
+            stream_session_id = normalize_agent_value(values.get("session.id"))
+            session_id = created_id or stream_session_id
+            if not session_id:
+                continue
+            record = remember(session_id)
+            _update_session_seen(record, timestamp)
+            if created_id:
+                parent_id = normalize_agent_value(values.get("parentID"))
+                record["parent_id"] = parent_id
+                record["title"] = normalize_agent_value(values.get("title")) or record.get("title")
+                record["agent"] = normalize_agent_value(values.get("agent")) or record.get("agent")
+                model = normalize_agent_value(values.get("model"))
+                if model:
+                    record["model"] = model
+                if parent_id is None and primary_session_id is None:
+                    primary_session_id = session_id
+            if stream_session_id:
+                record["provider"] = (
+                    normalize_agent_value(values.get("providerID"))
+                    or normalize_agent_value(values.get("llm.provider"))
+                    or record.get("provider")
+                )
+                record["model"] = (
+                    normalize_agent_value(values.get("modelID"))
+                    or normalize_agent_value(values.get("llm.model"))
+                    or record.get("model")
+                )
+                record["agent"] = normalize_agent_value(values.get("agent")) or record.get("agent")
+                record["mode"] = normalize_agent_value(values.get("mode")) or record.get("mode")
+                if primary_session_id is None:
+                    primary_session_id = session_id
+
+        sessions = [records[session_id] for session_id in order]
+        base = inspector_base_url.rstrip("/") or "http://127.0.0.1:25947"
+        for record in sessions:
+            record["inspector_url"] = f"{base}/?sessionId={urllib.parse.quote(str(record['session_id']))}"
+        if not primary_session_id and sessions:
+            primary_session_id = str(sessions[0]["session_id"])
+        return {
+            "schema": OPENCODE_SESSION_SCHEMA,
+            "generated_at": utc_now_seconds_text(),
+            "inspector_base_url": base,
+            "inspector_sessions_api": f"{base}/api/sessions",
+            "primary_session_id": primary_session_id,
+            "session_count": len(sessions),
+            "subagent_session_count": sum(1 for item in sessions if item.get("parent_id")),
+            "sessions": sessions,
+        }
+
+    def render_sessions_md(self, summary: dict[str, Any]) -> str:
+        from ..cli import md_artifact_external_link, md_cell
+
+        lines = [
+            f"# MiMo Code Sessions",
+            "",
+            f"- Generated at: {summary.get('generated_at')}",
+            f"- Inspector: {summary.get('inspector_base_url')}",
+            f"- Sessions API: {summary.get('inspector_sessions_api')}",
+            f"- Primary session: `{summary.get('primary_session_id') or ''}`",
+            f"- Session count: {summary.get('session_count', 0)}",
+            f"- Subagent session count: {summary.get('subagent_session_count', 0)}",
+            "",
+            "| session | parent | agent | mode | model | steps | tools | errors | inspector |",
+            "| --- | --- | --- | --- | --- | ---: | ---: | ---: | --- |",
+        ]
+        for item in summary.get("sessions", []) or []:
+            lines.append(
+                "| "
+                + " | ".join(
+                    [
+                        md_cell(item.get("session_id")),
+                        md_cell(item.get("parent_id")),
+                        md_cell(item.get("agent")),
+                        md_cell(item.get("mode")),
+                        md_cell(item.get("model")),
+                        md_cell(item.get("step_count")),
+                        md_cell(item.get("tool_call_count")),
+                        md_cell(item.get("error_count")),
+                        md_artifact_external_link(str(item.get("inspector_url") or ""), "open"),
+                    ]
+                )
+                + " |"
+            )
+        if not summary.get("sessions"):
+            lines.append("| none |  |  |  |  | 0 | 0 | 0 |  |")
+        if summary.get("inspector_backfill_status") and summary.get("inspector_backfill_status") != "SKIPPED":
+            lines.extend(
+                [
+                    "",
+                    "## Inspector Log Backfill",
+                    "",
+                    f"- Status: `{summary.get('inspector_backfill_status') or ''}`",
+                    f"- Query mode: `{summary.get('inspector_log_query_mode') or ''}`",
+                    f"- Logs URL: {md_artifact_external_link(str(summary.get('inspector_logs_url') or ''), 'open')}",
+                    f"- Log count: {summary.get('inspector_log_count', 0)}",
+                    f"- Preview count: {summary.get('inspector_log_preview_count', 0)} / {summary.get('inspector_log_preview_limit', 0)}",
+                ]
+            )
+            if summary.get("inspector_logs_artifact"):
+                lines.append(f"- Full log artifact: `{summary.get('inspector_logs_artifact')}`")
+            if summary.get("inspector_backfill_reason"):
+                lines.append(f"- Reason: {summary.get('inspector_backfill_reason')}")
+            inspector_logs = summary.get("inspector_logs") or []
+            if inspector_logs:
+                lines.extend(
+                    [
+                        "",
+                        "| log | session | model | status | path | inspector |",
+                        "| ---: | --- | --- | ---: | --- | --- |",
+                    ]
+                )
+                for item in inspector_logs:
+                    lines.append(
+                        "| "
+                        + " | ".join(
+                            [
+                                md_cell(item.get("id")),
+                                md_cell(item.get("session_id")),
+                                md_cell(item.get("model")),
+                                md_cell(item.get("status")),
+                                md_cell(item.get("path")),
+                                md_artifact_external_link(str(item.get("inspector_url") or ""), "open"),
+                            ]
+                        )
+                        + " |"
+                    )
+        lines.append("")
+        return "\n".join(lines)
+
+    # -- log parsing: model
+
+    def model_from_log(self, run_dir: Path) -> str:
+        log_path = run_dir / self.log_dir_name() / "opencode.log"
+        if not log_path.exists():
+            return ""
+        from ..cli import read_trace_text
+
+        try:
+            text = read_trace_text(log_path)
+        except OSError:
+            return ""
+        command_match = re.search(r"^\$\s+.+?\s--model\s+(\S+)", text, re.MULTILINE)
+        if command_match:
+            return command_match.group(1)
+        stream_match = re.search(r"providerID=(\S+)\s+modelID=(\S+)", text)
+        if stream_match:
+            return f"{stream_match.group(1)}/{stream_match.group(2)}"
+        runtime_match = re.search(r"llm\.provider=(\S+)\s+llm\.model=(\S+)", text)
+        if runtime_match:
+            return f"{runtime_match.group(1)}/{runtime_match.group(2)}"
+        return ""
+
+    # -- log parsing: activity
+
+    def extract_activity_from_text(self, text: str, *, tail_lines: int = 500) -> dict[str, Any]:
+        from ..cli import plain_log_text
+
+        lines = text.splitlines()[-tail_lines:]
+        sessions: set[str] = set()
+        subagent_sessions: set[str] = set()
+        touched_files: list[str] = []
+        latest: dict[str, Any] = {}
+        event_count = 0
+
+        def remember(
+            session_id: str | None,
+            *,
+            mode: str | None = None,
+            agent: str | None = None,
+            model: str | None = None,
+            message: str | None = None,
+            timestamp: str | None = None,
+        ) -> None:
+            nonlocal event_count, latest
+            normalized_session = normalize_agent_value(session_id)
+            if not normalized_session:
+                return
+            event_count += 1
+            sessions.add(normalized_session)
+            normalized_mode = normalize_agent_value(mode)
+            if normalized_mode == "subagent":
+                subagent_sessions.add(normalized_session)
+            latest = {
+                "latest_session_id": normalized_session,
+                "latest_mode": normalized_mode,
+                "latest_agent": normalize_agent_value(agent),
+                "latest_model": normalize_agent_value(model),
+                "latest_message": normalize_agent_value(message),
+                "latest_timestamp": normalize_agent_value(timestamp),
+            }
+
+        for raw_line in lines:
+            line = plain_log_text(raw_line).strip()
+            if not line:
+                continue
+            if line.startswith("{"):
+                try:
+                    payload = json.loads(line)
+                except json.JSONDecodeError:
+                    payload = None
+                if isinstance(payload, dict):
+                    part = payload.get("part") if isinstance(payload.get("part"), dict) else {}
+                    session_id = payload.get("sessionID") or part.get("sessionID")
+                    if not session_id:
+                        session_id = payload.get("turnID") or (part or {}).get("turnID")
+                    remember(
+                        str(session_id or ""),
+                        mode=str(part.get("mode", "") if isinstance(part, dict) else ""),
+                        agent=str(part.get("agent", "") if isinstance(part, dict) else ""),
+                        message=str(payload.get("type") or ""),
+                        timestamp=str(payload.get("timestamp") or ""),
+                    )
+                    continue
+
+            values = parse_log_key_values(line)
+            session_id = values.get("session.id")
+            if not session_id and "message=created" in line:
+                session_id = values.get("id")
+            if session_id:
+                remember(
+                    session_id,
+                    mode=values.get("mode"),
+                    agent=values.get("agent"),
+                    model=values.get("modelID") or values.get("llm.model") or values.get("model"),
+                    message=values.get("message"),
+                    timestamp=values.get("timestamp"),
+                )
+            touched = values.get("file") if values.get("message") == "touching file" else None
+            if touched:
+                touched_files.append(touched)
+
+        latest.update(
+            {
+                "event_count": event_count,
+                "session_count": len(sessions),
+                "subagent_session_count": len(subagent_sessions),
+                "has_subagent_activity": bool(subagent_sessions) or latest.get("latest_mode") == "subagent",
+                "touched_file_count": len(touched_files),
+                "latest_touched_file": touched_files[-1] if touched_files else "",
+            }
+        )
+        return latest
+
+
+def _mimocode_active_model(execution: dict[str, Any]) -> str:
+    for key in ("actual_model", "model"):
+        value = normalize_agent_value(str(execution.get(key) or ""))
+        if value and value != "inherit":
+            return value
+    return ""
+
+
+def _mimocode_set_option_before_message(command: list[str], flag: str, value: str) -> list[str]:
+    for index, item in enumerate(command):
+        text = str(item)
+        if text == flag:
+            if index + 1 >= len(command):
+                raise RuntimeError(f"{flag} is present without a following value")
+            result = list(command)
+            result[index + 1] = value
+            return result
+        if text.startswith(f"{flag}="):
+            result = list(command)
+            result[index] = f"{flag}={value}"
+            return result
+    insert_at = len(command)
+    for index, item in enumerate(command):
+        if str(item) == DEFAULT_ATTACHED_INSTRUCTION_PROMPT:
+            insert_at = index
+            break
+    return [*command[:insert_at], flag, value, *command[insert_at:]]
+
+
+# -- shared session helpers (same as opencode.py)
+
+def _ensure_session_record(
+    records: dict[str, dict[str, Any]], session_id: str
+) -> dict[str, Any]:
+    record = records.get(session_id)
+    if record is None:
+        record = {
+            "session_id": session_id,
+            "parent_id": None,
+            "agent": None,
+            "title": None,
+            "provider": None,
+            "model": None,
+            "mode": None,
+            "first_seen": None,
+            "last_seen": None,
+            "step_count": 0,
+            "tool_call_count": 0,
+            "text_part_count": 0,
+            "error_count": 0,
+            "tokens": {"input": 0, "output": 0, "reasoning": 0, "cache_read": 0, "cache_write": 0},
+        }
+        records[session_id] = record
+    return record
+
+
+def _update_session_seen(record: dict[str, Any], timestamp: str | None) -> None:
+    if not timestamp:
+        return
+    if record.get("first_seen") is None or str(timestamp) < str(record.get("first_seen")):
+        record["first_seen"] = timestamp
+    if record.get("last_seen") is None or str(timestamp) > str(record.get("last_seen")):
+        record["last_seen"] = timestamp
+
+
+def _session_timestamp_from_json(payload: dict[str, Any]) -> str | None:
+    import datetime as dt
+
+    value = payload.get("timestamp")
+    if isinstance(value, (int, float)):
+        try:
+            return dt.datetime.fromtimestamp(value / 1000, tz=dt.timezone.utc).isoformat()
+        except (OverflowError, OSError, ValueError):
+            return str(value)
+    if value is None:
+        return None
+    return str(value)
+
+
+def _apply_step_tokens(record: dict[str, Any], tokens: dict[str, Any]) -> None:
+    bucket = record.setdefault("tokens", {})
+    for key in ("input", "output", "reasoning"):
+        value = tokens.get(key)
+        if isinstance(value, int):
+            bucket[key] = int(bucket.get(key, 0)) + value
+    cache = tokens.get("cache")
+    if isinstance(cache, dict):
+        read = cache.get("read")
+        write = cache.get("write")
+        if isinstance(read, int):
+            bucket["cache_read"] = int(bucket.get("cache_read", 0)) + read
+        if isinstance(write, int):
+            bucket["cache_write"] = int(bucket.get("cache_write", 0)) + write