eval-harness 0.2.18

package/src/eval/runners/base.py

@@ -0,0 +1,197 @@
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Any
+
+
+DEFAULT_AGENT_PROMPT = "\n".join(
+    [
+        "Read and execute work/INSTRUCTION.md. Work autonomously, do not ask for human input, and write all required result artifacts.",
+        "",
+        "Harness execution contract:",
+        "- Treat AGENT_EVAL_TIMEOUT_MINUTES as the hard worker budget.",
+        "- Reserve at least AGENT_EVAL_FINALIZE_BEFORE_MINUTES minutes for finalization.",
+        "- Always finish by writing work/result/output.md, work/reports/FINAL_RESULT.json, and work/reports/FINAL_RESULT.md, even when blocked or partial.",
+        "- Workspace layout: current directory is the evaluation workspace; task code is under project/ and the submitted work package is under work/.",
+        "- Runtime artifacts must stay under work/result, work/reports, and work/logs/trace. Do not use root-level result/, reports/, or logs/trace/ paths.",
+        "- When delegating to subagents, pass workspace-relative work artifact paths with the work/ prefix.",
+        "- Keep large audits chunked: read existing handoff/index artifacts first, process only a small batch of source files or flow families per turn, and write progress artifacts before continuing.",
+        "- Do not run helper/finalization scripts through truncating pipes such as Select-Object -First or head. Write full helper output to work/logs/trace first, then preview that saved log separately.",
+        "- Before final response, verify the required work/result, work/reports, and work/logs/trace artifacts exist under the work/ prefix.",
+        "- Final agent responses should be concise status plus artifact paths; put large reports in work/reports or work/logs/trace.",
+        "- If running on Windows, write Java and source files as UTF-8 without BOM.",
+        "- Avoid PowerShell Out-File -Encoding utf8 for Java/source files; use utf8NoBOM or an explicit UTF8Encoding(false) writer.",
+    ]
+)
+
+
+DEFAULT_ATTACHED_INSTRUCTION_PROMPT = DEFAULT_AGENT_PROMPT.replace(
+    "work/INSTRUCTION.md", "the attached INSTRUCTION.md", 1
+)
+
+
+class AgentRunner:
+    """Abstract base for agent runner adapters (OpenCode, MiMo Code, etc.)."""
+
+    # -- identity
+
+    @property
+    def name(self) -> str:
+        """Machine-readable runner identifier (e.g. 'opencode', 'mimocode')."""
+        raise NotImplementedError
+
+    @property
+    def binary(self) -> str:
+        """CLI binary name (e.g. 'opencode', 'mimo')."""
+        raise NotImplementedError
+
+    # -- config keys used in config JSON
+
+    @property
+    def config_key(self) -> str:
+        """Key in config for runner-specific settings (e.g. 'opencode', 'mimocode')."""
+        raise NotImplementedError
+
+    # -- command building
+
+    def default_command(self, config: dict[str, Any]) -> list[str]:
+        """Default CLI command template."""
+        raise NotImplementedError
+
+    def command_with_model(self, command: list[str], model: str) -> list[str]:
+        """Inject --model into the command."""
+        return _command_set_named_arg(command, "--model", model)
+
+    def command_with_variant(self, command: list[str], variant: str) -> list[str]:
+        """Inject the variant/agent flag. Override per runner."""
+        return command
+
+    def model_from_command(self, command: list[str]) -> str:
+        """Extract model from command line."""
+        return _command_get_named_arg(command, "--model")
+
+    # -- log directory
+
+    def log_dir_name(self) -> str:
+        """Subdirectory under run_dir for agent logs/sessions/config."""
+        raise NotImplementedError
+
+    # -- asset installation
+
+    def default_asset_installs(self) -> list[dict[str, Any]]:
+        """Default asset installation specs (skills, agents, versions)."""
+        raise NotImplementedError
+
+    # -- runtime profile seeding
+
+    def runtime_profile_candidates(
+        self, base_env: dict[str, str], runtime: dict[str, str]
+    ) -> list[tuple[Path | None, Path, str]]:
+        """Return (source, dest, label) tuples for seeding runtime profile."""
+        raise NotImplementedError
+
+    # -- provider config
+
+    def write_agent_config(self, manifest: dict[str, Any]) -> str | None:
+        """Write agent-specific config file. Returns config path or None."""
+        return None
+
+    # -- log parsing: sessions
+
+    def extract_sessions_from_text(
+        self, text: str, *, inspector_base_url: str = "http://127.0.0.1:25947"
+    ) -> dict[str, Any]:
+        """Parse agent log text into session summary."""
+        raise NotImplementedError
+
+    def render_sessions_md(self, summary: dict[str, Any]) -> str:
+        """Render session summary as Markdown."""
+        raise NotImplementedError
+
+    # -- log parsing: model
+
+    def model_from_log(self, run_dir: Path) -> str:
+        """Extract model name from log file."""
+        raise NotImplementedError
+
+    # -- log parsing: activity
+
+    def extract_activity_from_text(self, text: str, *, tail_lines: int = 500) -> dict[str, Any]:
+        """Extract recent agent activity from log text."""
+        raise NotImplementedError
+
+    # -- CLI flags
+
+    def format_flags(self) -> list[str]:
+        """Flags for JSON output format."""
+        return ["--format", "json", "--print-logs", "--log-level", "INFO"]
+
+    def variant_cli_flag(self) -> str:
+        """Name of the CLI flag for variant/agent selection (e.g. '--variant', '--agent')."""
+        return ""
+
+    def variant_from_command(self, command: list[str]) -> str:
+        """Extract variant/agent from command line."""
+        flag = self.variant_cli_flag()
+        if not flag:
+            return ""
+        return _command_get_named_arg(command, flag)
+
+    def default_worker_tools(self, config: dict[str, Any] | None = None) -> list[str]:
+        """Executables Jenkins should check before starting real workers."""
+        return [self.binary, "node"]
+
+
+# -- helpers used by runners
+
+def _command_get_named_arg(command: list[str], flag: str) -> str:
+    for index, item in enumerate(command):
+        text = str(item)
+        if text == flag and index + 1 < len(command):
+            return str(command[index + 1])
+        if text.startswith(f"{flag}="):
+            return text.split("=", 1)[1]
+    return ""
+
+
+def _command_set_named_arg(command: list[str], flag: str, value: str) -> list[str]:
+    for index, item in enumerate(command):
+        text = str(item)
+        if text == flag:
+            if index + 1 >= len(command):
+                raise RuntimeError(f"{flag} is present without a following value")
+            result = list(command)
+            result[index + 1] = value
+            return result
+        if text.startswith(f"{flag}="):
+            result = list(command)
+            result[index] = f"{flag}={value}"
+            return result
+    return [*command, flag, value]
+
+
+def parse_log_key_values(line: str) -> dict[str, str]:
+    """Parse key=value pairs from a log line. Shared by all runners."""
+    import json
+
+    values: dict[str, str] = {}
+    pattern = re.compile(r'([A-Za-z_][A-Za-z0-9_.-]*)=("(?:\\.|[^"])*"|[^\s]+)')
+    for match in pattern.finditer(line):
+        raw = match.group(2)
+        if raw.startswith('"') and raw.endswith('"'):
+            try:
+                raw = json.loads(raw)
+            except json.JSONDecodeError:
+                raw = raw[1:-1]
+        values[match.group(1)] = str(raw)
+    return values
+
+
+def normalize_agent_value(value: str | None) -> str | None:
+    if value is None:
+        return None
+    text = str(value).strip()
+    if not text or text == "undefined":
+        return None
+    return text