@rulemetric/cli 0.13.0 → 0.14.0

package/dist/harbor/rm_host_claude.py

@@ -0,0 +1,264 @@
+"""Harbor agent that runs the USER'S OWN Claude Code on the host.
+
+No API key or token: the host `claude` CLI uses the user's existing
+subscription login (macOS keychain). The agent downloads the task's /app from
+the environment, runs `claude -p <instruction>` in a host temp dir, and
+uploads the result back for in-container verification.
+
+Instruction delivery (the RuleMetric with-arm) via agent kwargs — all
+host-side; the container never sees the injected files:
+
+    --ak claude_md=/abs/path/CLAUDE.md      claude_md delivery (workdir root)
+    --ak skills_dirs=/abs/a,/abs/b          skill delivery (.claude/skills/<name>)
+    --ak append_system_prompt="..."         prompt delivery (--append-system-prompt)
+    --ak container_shell=true               route ALL shell commands into the
+                                            task container via the MCP proxy
+                                            (mcp_container_shell.py); host
+                                            Bash tool is disabled
+
+container_shell requires the local docker environment (container resolved by
+compose labels) and imports two harbor internals pinned to 0.22.x:
+_sanitize_docker_compose_project_name and MAIN_SERVICE_NAME.
+"""
+
+import asyncio
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+import provenance
+
+from harbor.agents.base import BaseAgent
+from harbor.environments.base import BaseEnvironment
+from harbor.models.agent.context import AgentContext
+
+_CONTAINER_SHELL_PROMPT = (
+    "Your shell is the task container, not this machine: run every shell "
+    "command with the mcp__container__bash tool (the host Bash tool is "
+    "disabled). The container's /app is your current working directory: "
+    "when using file tools, always use paths RELATIVE to the working "
+    "directory (never absolute /app/... paths) — they are synced into the "
+    "container's /app before each command runs and when you finish. "
+    "Alternatively, write files directly through the container shell."
+)
+
+
+class HostClaudeCode(BaseAgent):
+    @staticmethod
+    def name() -> str:
+        return "host-claude-code"
+
+    def __init__(
+        self,
+        logs_dir: Path,
+        model_name: str | None = None,
+        claude_md: str | None = None,
+        skills_dirs: str | None = None,
+        append_system_prompt: str | None = None,
+        container_shell: str | bool | None = None,
+        agent_timeout_sec: float | None = None,
+        **kwargs,
+    ):
+        super().__init__(logs_dir=logs_dir, model_name=model_name, **kwargs)
+        self._claude_md = claude_md
+        self._skills_dirs = [s for s in (skills_dirs or "").split(",") if s]
+        self._append_system_prompt = append_system_prompt
+        self._container_shell = str(container_shell).lower() in ("true", "1", "yes")
+        self._timeout = agent_timeout_sec or 600.0
+
+    def version(self) -> str:
+        return "0.2.0"
+
+    def _delivery(self) -> list[str]:
+        """Which delivery channels this arm actually used, for the stamp."""
+        used = []
+        if self._claude_md:
+            used.append("claude_md")
+        if self._skills_dirs:
+            used.append("skills_dirs")
+        if self._append_system_prompt:
+            used.append("append_system_prompt")
+        if self._container_shell:
+            used.append("container_shell")
+        return used
+
+    async def setup(self, environment: BaseEnvironment) -> None:
+        if shutil.which("claude") is None:
+            raise RuntimeError("host `claude` CLI not found on PATH")
+
+    def _resolve_container_id(self, environment: BaseEnvironment) -> str:
+        # Pinned harbor 0.22.x internals; the compose labels are stable Docker
+        # behavior, only the project-name derivation is harbor's.
+        from harbor.constants import MAIN_SERVICE_NAME
+        from harbor.environments.docker.docker import (
+            _sanitize_docker_compose_project_name,
+        )
+
+        project = _sanitize_docker_compose_project_name(environment.session_id)
+        proc = subprocess.run(
+            [
+                "docker",
+                "ps",
+                "-q",
+                "--filter",
+                f"label=com.docker.compose.project={project}",
+                "--filter",
+                f"label=com.docker.compose.service={MAIN_SERVICE_NAME}",
+            ],
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        container_id = proc.stdout.strip().splitlines()
+        if len(container_id) != 1:
+            raise RuntimeError(
+                f"expected exactly one container for project {project}, "
+                f"got {container_id!r} (container_shell needs the local "
+                "docker environment)"
+            )
+        return container_id[0]
+
+    def _inject(self, workdir: Path) -> list[Path]:
+        """Copy instruction artifacts into the workdir; return paths to strip
+        before the final upload so they never count as agent output."""
+        injected: list[Path] = []
+        if self._claude_md:
+            target = workdir / "CLAUDE.md"
+            shutil.copyfile(self._claude_md, target)
+            injected.append(target)
+        if self._skills_dirs:
+            skills_root = workdir / ".claude"
+            for src in self._skills_dirs:
+                src_path = Path(src)
+                shutil.copytree(
+                    src_path,
+                    skills_root / "skills" / src_path.name,
+                    dirs_exist_ok=True,
+                )
+            injected.append(skills_root)
+        return injected
+
+    def _build_command(self, instruction: str, environment: BaseEnvironment,
+                       workdir: Path) -> list[str]:
+        cmd = [
+            "claude",
+            "-p",
+            instruction,
+            "--permission-mode",
+            "acceptEdits",
+            "--output-format",
+            "json",
+        ]
+        if self.model_name:
+            cmd += ["--model", self.model_name.split("/")[-1]]
+        if not self._container_shell:
+            # acceptEdits auto-approves file edits but NOT Bash, and a task that
+            # invites "write a script, then run it" stalls at "please approve the
+            # Bash command" and uploads nothing -- measured 2026-08-31: 12 of 15
+            # rm-batch-task with-arm trials died exactly there, reading as rule
+            # non-compliance. The workdir is a per-trial temp dir; Bash there is
+            # what any real session has. (The container path keeps Bash disallowed
+            # by design -- the MCP container shell replaces it.)
+            cmd += ["--allowedTools", "Bash"]
+
+        system_prompt_parts = []
+        if self._container_shell:
+            container_id = self._resolve_container_id(environment)
+            mcp_config = {
+                "mcpServers": {
+                    "container": {
+                        "command": sys.executable,
+                        "args": [str(Path(__file__).parent / "mcp_container_shell.py")],
+                        "env": {
+                            "RM_CONTAINER_ID": container_id,
+                            "RM_WORKDIR": str(workdir),
+                        },
+                    }
+                }
+            }
+            mcp_path = self.logs_dir / "mcp-config.json"
+            mcp_path.write_text(json.dumps(mcp_config))
+            cmd += [
+                "--mcp-config",
+                str(mcp_path),
+                "--strict-mcp-config",
+                "--disallowedTools",
+                "Bash",
+                "--allowedTools",
+                "mcp__container__bash",
+            ]
+            system_prompt_parts.append(_CONTAINER_SHELL_PROMPT)
+        if self._append_system_prompt:
+            system_prompt_parts.append(self._append_system_prompt)
+        if system_prompt_parts:
+            cmd += ["--append-system-prompt", "\n\n".join(system_prompt_parts)]
+        return cmd
+
+    async def run(
+        self,
+        instruction: str,
+        environment: BaseEnvironment,
+        context: AgentContext,
+    ) -> None:
+        workdir = Path(tempfile.mkdtemp(prefix="harbor-host-claude-"))
+        try:
+            await environment.download_dir("/app", workdir)
+            injected = self._inject(workdir)
+            cmd = self._build_command(instruction, environment, workdir)
+
+            env = {
+                k: v
+                for k, v in os.environ.items()
+                # Drop nested-session markers so the child runs as a fresh CLI.
+                if not k.startswith(("CLAUDECODE", "CLAUDE_CODE_"))
+            }
+            proc = await asyncio.create_subprocess_exec(
+                *cmd,
+                cwd=workdir,
+                env=env,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+            )
+            try:
+                stdout, stderr = await asyncio.wait_for(
+                    proc.communicate(), timeout=self._timeout
+                )
+            except asyncio.TimeoutError:
+                proc.kill()
+                raise
+            (self.logs_dir / "claude-output.json").write_bytes(stdout)
+            (self.logs_dir / "claude-stderr.txt").write_bytes(stderr)
+            # Written for EVERY trial, including one about to raise below: the
+            # provenance of a failed trial is exactly what tells a later reader
+            # whether the failure was the rule or the harness (#415).
+            provenance.write_stamp(
+                self.logs_dir,
+                dict(
+                    provenance.stamp(
+                        cmd,
+                        observed_model=provenance.observed_model_from_claude(stdout),
+                        engine="claude",
+                        engine_version=provenance.cli_version("claude"),
+                        env=env,
+                        delivery=self._delivery(),
+                    ),
+                    agent_version=self.version(),
+                ),
+            )
+            if proc.returncode != 0:
+                raise RuntimeError(
+                    f"claude exited {proc.returncode}: {stderr.decode()[:2000]}"
+                )
+
+            for path in injected:
+                if path.is_dir():
+                    shutil.rmtree(path, ignore_errors=True)
+                else:
+                    path.unlink(missing_ok=True)
+            await environment.upload_dir(workdir, "/app")
+        finally:
+            shutil.rmtree(workdir, ignore_errors=True)

package/dist/harbor/rm_host_codex.py

@@ -0,0 +1,208 @@
+"""Harbor agent that runs the USER'S OWN Codex CLI on the host.
+
+The cross-vendor twin of rm_host_claude.py. No API key: the host `codex` CLI
+uses the user's existing ChatGPT login. The agent downloads the task's /app,
+runs `codex exec` in a host temp dir, and uploads the result back for
+in-container verification.
+
+Why a separate agent rather than a flag on the claude one: the two CLIs differ
+in the three things that matter to an A/B arm — which instruction file the
+agent reads (AGENTS.md vs CLAUDE.md), whether there is a system-prompt channel
+(codex has none; the preamble is prepended to the instruction), and how the
+final message comes back (a file via -o, not a JSON blob on stdout). Papering
+over that with `if engine == "codex"` inside one agent is how an arm silently
+delivers an instruction the agent never reads and the trial still reports a
+clean result.
+
+Instruction delivery (the RuleMetric with-arm) via agent kwargs:
+
+    --ak agents_md=/abs/path/AGENTS.md      instruction delivery (workdir root)
+    --ak append_system_prompt="..."         prepended to the instruction
+
+Flags verified against codex-cli 0.144.0. `codex -q` was removed from the CLI
+in an earlier version and left a mapping that could never succeed, so every
+flag here was probed against the installed binary rather than remembered.
+"""
+
+import asyncio
+import os
+import shutil
+import tempfile
+from pathlib import Path
+
+import provenance
+
+from harbor.agents.base import BaseAgent
+from harbor.environments.base import BaseEnvironment
+from harbor.models.agent.context import AgentContext
+
+
+class HostCodex(BaseAgent):
+    @staticmethod
+    def name() -> str:
+        return "host-codex"
+
+    def __init__(
+        self,
+        logs_dir: Path,
+        model_name: str | None = None,
+        agents_md: str | None = None,
+        append_system_prompt: str | None = None,
+        skills_dirs: str | None = None,
+        container_shell: str | bool | None = None,
+        agent_timeout_sec: float | None = None,
+        **kwargs,
+    ):
+        super().__init__(logs_dir=logs_dir, model_name=model_name, **kwargs)
+        self._agents_md = agents_md
+        self._append_system_prompt = append_system_prompt
+        self._timeout = agent_timeout_sec or 600.0
+
+        # Refuse, loudly, rather than ignore. An arm that asked for skill
+        # delivery or a container shell and got neither is not a weaker arm —
+        # it is a DIFFERENT experiment than the one the operator described, and
+        # its result would be compared against a claude arm that did get them.
+        if skills_dirs:
+            raise ValueError(
+                "skills_dirs is Claude-Code-specific (.claude/skills) and has no "
+                "codex equivalent; deliver the same guidance via --ak agents_md "
+                "or --ak append_system_prompt so the arms stay comparable."
+            )
+        if str(container_shell).lower() in ("true", "1", "yes"):
+            raise ValueError(
+                "container_shell is not implemented for the codex agent (it needs "
+                "codex's own MCP config shape, unverified here). Run the codex arm "
+                "without it, or use the claude agent for container-shell trials."
+            )
+
+    def version(self) -> str:
+        return "0.1.0"
+
+    def _delivery(self) -> list[str]:
+        """Which delivery channels this arm actually used, for the stamp."""
+        used = []
+        if self._agents_md:
+            used.append("agents_md")
+        if self._append_system_prompt:
+            used.append("append_system_prompt")
+        return used
+
+    async def setup(self, environment: BaseEnvironment) -> None:
+        if shutil.which("codex") is None:
+            raise RuntimeError("host `codex` CLI not found on PATH")
+
+    def _inject(self, workdir: Path) -> list[Path]:
+        """Copy instruction artifacts into the workdir; return paths to strip
+        before the final upload so they never count as agent output."""
+        injected: list[Path] = []
+        if self._agents_md:
+            target = workdir / "AGENTS.md"
+            shutil.copyfile(self._agents_md, target)
+            injected.append(target)
+        return injected
+
+    def _build_command(self, last_message_file: Path) -> list[str]:
+        cmd = [
+            "codex",
+            "exec",
+            # JSONL on stdout so the run has token telemetry at all. Without it
+            # codex-output.txt was plain progress text with no usage envelope,
+            # every codex arm took summarize.py's null-cost branch, and a codex
+            # A/B was indistinguishable from a free run. The ANSWER still comes
+            # from --output-last-message; stdout is parsed only for usage.
+            "--json",
+            # The agent must be able to WRITE — this is a task run, not the
+            # read-only analysis path in codex-exec.ts.
+            "--sandbox",
+            "workspace-write",
+            # The per-trial workdir is a temp dir, not a git repo.
+            "--skip-git-repo-check",
+            # Keep synthetic benchmark trials out of ~/.codex/sessions: they are
+            # not the user's work, and letting them land there would contaminate
+            # the very session history RuleMetric measures.
+            "--ephemeral",
+            "--color",
+            "never",
+            "--output-last-message",
+            str(last_message_file),
+        ]
+        if self.model_name:
+            cmd += ["--model", self.model_name.split("/")[-1]]
+        # Prompt on stdin: instructions embed whole task descriptions and argv
+        # has an ARG_MAX ceiling.
+        cmd.append("-")
+        return cmd
+
+    async def run(
+        self,
+        instruction: str,
+        environment: BaseEnvironment,
+        context: AgentContext,
+    ) -> None:
+        workdir = Path(tempfile.mkdtemp(prefix="harbor-host-codex-"))
+        last_message = self.logs_dir / "codex-last-message.txt"
+        try:
+            await environment.download_dir("/app", workdir)
+            injected = self._inject(workdir)
+
+            # Codex has no --append-system-prompt channel, so the preamble is
+            # prepended. Same delivery as the eval path in codex-exec.ts.
+            prompt = instruction
+            if self._append_system_prompt:
+                prompt = f"{self._append_system_prompt}\n\n{instruction}"
+
+            cmd = self._build_command(last_message)
+            env = {
+                k: v
+                for k, v in os.environ.items()
+                # Drop nested-session markers so the child runs as a fresh CLI,
+                # and never route a trial through the capture proxy.
+                if not k.startswith(("CLAUDECODE", "CLAUDE_CODE_"))
+                and k not in ("HTTPS_PROXY", "HTTP_PROXY", "https_proxy", "http_proxy")
+            }
+            proc = await asyncio.create_subprocess_exec(
+                *cmd,
+                cwd=workdir,
+                env=env,
+                stdin=asyncio.subprocess.PIPE,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+            )
+            try:
+                stdout, stderr = await asyncio.wait_for(
+                    proc.communicate(prompt.encode()), timeout=self._timeout
+                )
+            except asyncio.TimeoutError:
+                proc.kill()
+                raise
+            (self.logs_dir / "codex-output.txt").write_bytes(stdout)
+            (self.logs_dir / "codex-stderr.txt").write_bytes(stderr)
+            # codex states no modelUsage envelope, so the stamp records the
+            # REQUESTED model and says so (model_source: 'requested') rather
+            # than presenting a flag as an observation.
+            provenance.write_stamp(
+                self.logs_dir,
+                dict(
+                    provenance.stamp(
+                        cmd,
+                        engine="codex",
+                        engine_version=provenance.cli_version("codex"),
+                        env=env,
+                        delivery=self._delivery(),
+                    ),
+                    agent_version=self.version(),
+                ),
+            )
+            if proc.returncode != 0:
+                raise RuntimeError(
+                    f"codex exited {proc.returncode}: {stderr.decode()[:2000]}"
+                )
+
+            for path in injected:
+                if path.is_dir():
+                    shutil.rmtree(path, ignore_errors=True)
+                else:
+                    path.unlink(missing_ok=True)
+            await environment.upload_dir(workdir, "/app")
+        finally:
+            shutil.rmtree(workdir, ignore_errors=True)