agent-hitch 0.2.7 → 0.2.8

package/integrations/harbor/hitch_harbor_environment.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import asyncio
 import json
 import re
 import tempfile
@@ -35,6 +36,8 @@ _GPU_OVERRIDE_PREFIX = "__HITCH_GPU_OVERRIDE_"
 class HitchHarborDockerEnvironment(DockerEnvironment):
     """Use Harbor's Docker semantics with a final ownership-label overlay."""
 
+    _hitch_static_no_network = False
+
     def __init__(
         self,
         *args: Any,
@@ -62,12 +65,16 @@ class HitchHarborDockerEnvironment(DockerEnvironment):
         self._hitch_main_gpu_count = hitch_main_gpu_count
         self._hitch_ownership_temp_dir: tempfile.TemporaryDirectory[str] | None = None
         self._hitch_ownership_compose_path: Path | None = None
+        self._hitch_phase_compose_path: Path | None = None
+        self._hitch_candidate_recycler = None
         positional = list(args)
         task_env_config = kwargs.get("task_env_config")
         task_env_position: int | None = None
         if task_env_config is None and len(positional) >= 5:
             task_env_position = 4
             task_env_config = positional[task_env_position]
+        self._hitch_static_no_network = _static_no_network(positional, kwargs, task_env_config)
+        self._hitch_static_network_receipt = None
         if task_env_config is not None:
             requested = getattr(task_env_config, "docker_image", None)
             resolved = self._hitch_prebuilt_task_image or self._hitch_resolved_images.get(requested)
@@ -78,6 +85,14 @@ class HitchHarborDockerEnvironment(DockerEnvironment):
                 else:
                     positional[task_env_position] = updated
         super().__init__(*positional, **kwargs)
+        self._hitch_benchmark = None
+        self._hitch_benchmark_platform = None
+        directory = getattr(self, "environment_dir", None)
+        if directory and (directory.parent / ".hitch-benchmark.json").is_file():
+            self._hitch_benchmark_platform = "linux/amd64"
+        if directory and directory.name == "environment" and (directory.parent / ".hitch-benchmark.json").is_file():
+            from hitch_benchmark import BenchmarkSession, descriptor
+            self._hitch_benchmark = BenchmarkSession(self, descriptor(directory))
         if (
             self._hitch_ownership_labels
             or self._hitch_service_resource_limits
@@ -85,16 +100,108 @@ class HitchHarborDockerEnvironment(DockerEnvironment):
             or self._hitch_prebuilt_task_image
             or self._hitch_model_proxy_host_gateway
             or self._hitch_main_gpu_count > 0
+            or self._hitch_benchmark_platform
+            or self._hitch_static_no_network
         ):
             self._hitch_ownership_compose_path = self._write_ownership_overlay()
 
     @property
     def _docker_compose_paths(self) -> list[Path]:
+        if self._hitch_static_no_network and (self._environment_docker_compose_path.exists() or self.extra_docker_compose_paths):
+            raise ValueError("static no-network requires a single Dockerfile/image environment without Compose additions")
         paths = list(super()._docker_compose_paths)
         if self._hitch_ownership_compose_path is not None:
             paths.append(self._hitch_ownership_compose_path)
+        if self._hitch_phase_compose_path is not None:
+            paths.append(self._hitch_phase_compose_path)
         return paths
 
+    def _requires_egress_control(self, *, startup_network_policy, phase_network_policies):
+        if self._hitch_static_no_network:
+            return False
+        return super()._requires_egress_control(startup_network_policy=startup_network_policy, phase_network_policies=phase_network_policies)
+
+    @property
+    def capabilities(self):
+        result = super().capabilities
+        if self._hitch_static_no_network:
+            return result.model_copy(update={"disable_internet": True, "dynamic_network_policy": False})
+        return result
+
+    async def _apply_network_policy(self, network_policy):
+        if self._hitch_static_no_network:
+            if network_policy != self.network_policy:
+                raise ValueError("static no-network cannot change runtime network policy")
+            return
+        await super()._apply_network_policy(network_policy)
+
+    async def _verify_static_network(self):
+        result = await self._run_docker_compose_command(["ps", "--all", "--quiet", MAIN_SERVICE_NAME])
+        identity = result.stdout.strip()
+        if not re.fullmatch(r"[a-f0-9]{64}", identity):
+            raise RuntimeError("static no-network requires exactly one owned main container")
+        template = '{"mode":{{json .HostConfig.NetworkMode}},"networks":{{json .NetworkSettings.Networks}},"ports":{{json .HostConfig.PortBindings}}}'
+        process = await asyncio.create_subprocess_exec("docker", "inspect", "--format", template, identity,
+                                                       stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL)
+        try:
+            stdout, _ = await asyncio.wait_for(process.communicate(), 15)
+        finally:
+            if process.returncode is None:
+                process.kill(); await process.wait()
+        if process.returncode != 0 or len(stdout) > 65536:
+            raise RuntimeError("could not verify static no-network container")
+        inspected = json.loads(stdout)
+        if inspected.get("mode") != "none" or set(inspected.get("networks") or {}) != {"none"} or inspected.get("ports"):
+            raise RuntimeError("Docker did not enforce the static no-network declaration")
+        self._hitch_static_network_receipt = {"container_id": identity, "network_mode": "none", "host_ports": False}
+
+    async def recycle_candidate_phase(self, phase_index: int) -> dict[str, Any]:
+        """Replace only main after the supervisor has exported the retired run.
+
+        Does not start a model or prepare a tool binding. The native task VM and
+        service containers remain live. Failures require whole-trial cleanup.
+        """
+        if self._hitch_candidate_recycler is None:
+            from hitch_candidate_recycle import CandidateRecycler
+            self._hitch_candidate_recycler = CandidateRecycler(self)
+        return await self._hitch_candidate_recycler.recycle(phase_index)
+
+    async def start(self, force_build: bool):
+        try:
+            await super().start(force_build)
+            if self._hitch_static_no_network:
+                await self._verify_static_network()
+            if self._hitch_benchmark:
+                await self._hitch_benchmark.prepare()
+        except BaseException:
+            # Harbor may abort setup before entering its normal teardown path.
+            try:
+                await self.stop(delete=True)
+            except BaseException:
+                pass
+            raise
+
+    async def stop_service(self, service: str) -> None:
+        try:
+            await super().stop_service(service)
+            if service == MAIN_SERVICE_NAME and self._hitch_benchmark:
+                await self._hitch_benchmark.snapshot()
+        except BaseException as error:
+            if self._hitch_benchmark:
+                self._hitch_benchmark.failure = {"phase": "quiesce", "error": type(error).__name__}
+                self._hitch_benchmark.write_journal()
+            raise
+
+    async def stop(self, delete: bool):
+        try:
+            if self._hitch_benchmark and self._hitch_benchmark.started:
+                try:
+                    await super().stop_service(MAIN_SERVICE_NAME)
+                finally:
+                    await self._hitch_benchmark.cleanup()
+        finally:
+            await super().stop(delete)
+
     def _write_ownership_overlay(self) -> Path:
         services = {MAIN_SERVICE_NAME}
         if self._enable_egress_control:
@@ -104,6 +211,7 @@ class HitchHarborDockerEnvironment(DockerEnvironment):
         external_networks: set[str] = set()
         external_volumes: set[str] = set()
         build_services: set[str] = set()
+        declared_platforms: set[str] = set()
         image_overlays: dict[str, str] = {}
         sources = [self._environment_docker_compose_path, *self.extra_docker_compose_paths]
         for source in sources:
@@ -117,6 +225,11 @@ class HitchHarborDockerEnvironment(DockerEnvironment):
             services.update(_mapping_names(document.get("services"), "services", source))
             for name, config in (document.get("services") or {}).items():
                 if isinstance(config, dict):
+                    if 'platform' in config:
+                        if config['platform'] is not None:
+                            declared_platforms.add(name)
+                        else:
+                            declared_platforms.discard(name)
                     if "build" in config and config.get("build") is not None:
                         build_services.add(name)
                     reference = config.get("image")
@@ -132,8 +245,14 @@ class HitchHarborDockerEnvironment(DockerEnvironment):
         service_overlays: dict[str, dict[str, Any]] = {}
         for name in sorted(services):
             config: dict[str, Any] = {}
+            # The benchmark default must not replace explicit per-service
+            # architecture, e.g. an ARM QEMU sidecar beside an AMD64 candidate.
+            if getattr(self, "_hitch_benchmark_platform", None) and name not in declared_platforms:
+                config["platform"] = self._hitch_benchmark_platform
             if labels:
                 config["labels"] = labels
+            if name == MAIN_SERVICE_NAME and self._hitch_static_no_network:
+                config["network_mode"] = "none"
             resolved_image = image_overlays.get(name)
             if name == MAIN_SERVICE_NAME and self._hitch_prebuilt_task_image:
                 resolved_image = self._hitch_prebuilt_task_image
@@ -207,6 +326,25 @@ class HitchHarborDockerEnvironment(DockerEnvironment):
         return target
 
 
+def _static_no_network(positional, kwargs, task_env_config) -> bool:
+    """Use Docker's isolated namespace only when no phase needs connectivity.
+
+    Compose tasks and dynamic policies retain Harbor's egress-control path.
+    A single image/Dockerfile has no sibling-service network to preserve.
+    """
+    policy = kwargs.get("network_policy", positional[6] if len(positional) > 6 else None)
+    phases = kwargs.get("phase_network_policies", positional[7] if len(positional) > 7 else ()) or ()
+    directory = kwargs.get("environment_dir", positional[0] if positional else None)
+    return bool(
+        getattr(getattr(task_env_config, "os", None), "value", None) == "linux"
+        and getattr(getattr(policy, "network_mode", None), "value", None) == "no-network"
+        and all(phase == policy for phase in phases)
+        and directory is not None
+        and not (Path(directory) / "docker-compose.yaml").exists()
+        and not kwargs.get("extra_docker_compose")
+    )
+
+
 def _mapping_names(value: Any, label: str, source: Path) -> set[str]:
     if value is None:
         return set()

package/integrations/harbor/hitch_harbor_verifier.py

@@ -23,9 +23,15 @@ from harbor.verifier.verifier import Verifier
 
 
 MAX_LOG_BYTES = 256 * 1024
+HITCH_AGENT_OUTCOME_NAME = "hitch-agent-outcome.json"
 LOG_NAMES = ("test-stdout.txt", "test-stderr.txt", "stdout.txt", "stderr.txt")
-OUTPUT_NAMES = (*LOG_NAMES, "reward.txt", "reward.json", "ctrf.json")
-CONTROL_NAMES = ("infrastructure-error.json", "infrastructure-retry-history.json")
+OUTPUT_NAMES = (*LOG_NAMES, "reward.txt", "reward.json", "process.json", "feedback.json", "ctrf.json")
+CONTROL_NAMES = (
+    "infrastructure-error.json",
+    "infrastructure-retry-history.json",
+    "candidate-ineligible.json",
+    "eligibility-gate.json",
+)
 
 INFRASTRUCTURE_PATTERNS: tuple[tuple[str, tuple[re.Pattern[str], ...]], ...] = (
     (
@@ -126,8 +132,28 @@ class HitchRetryingVerifier(Verifier):
             self._write_phase_timing(started_ns)
 
     async def _verify_with_retries(self) -> VerifierResult:
+        directory = getattr(getattr(getattr(self, "task", None), "paths", None), "environment_dir", None)
+        if directory and (directory.parent / ".hitch-benchmark.json").is_file():
+            from hitch_benchmark import restore_final_response, validate_collected_submission
+            validate_collected_submission(self)
+            await restore_final_response(self)
         # Do not trust a control file left by the candidate or a prior phase.
         await self._remove_files(CONTROL_NAMES)
+        eligibility = self._read_trusted_agent_outcome()
+        if eligibility is None:
+            self._write_json("eligibility-gate.json", {
+                "schema_version": "1", "status": "unavailable", "verifier_executed": True,
+            })
+        elif eligibility["gradeability"] == "ungradeable":
+            self._write_json("candidate-ineligible.json", {
+                "schema_version": "1",
+                "code": "candidate_evidence_unavailable",
+                "run_id": eligibility["run_id"],
+                "candidate_bundle": eligibility["candidate_bundle"],
+                "reason_code": eligibility.get("reason_code", "candidate_evidence_unavailable"),
+                "verifier_executed": False,
+            })
+            return VerifierResult(rewards={"reward": 0})
         attempts: list[dict[str, Any]] = []
         for attempt in range(1, self.infrastructure_retries + 2):
             result: VerifierResult | None = None
@@ -152,6 +178,10 @@ class HitchRetryingVerifier(Verifier):
                 if attempts:
                     self._write_history("recovered", attempts)
                 assert result is not None
+                directory = getattr(getattr(getattr(self, "task", None), "paths", None), "environment_dir", None)
+                if directory and (directory.parent / ".hitch-benchmark.json").is_file():
+                    from hitch_benchmark import normalize_rewards
+                    return normalize_rewards(self, result)
                 return result
 
             attempts.append(
@@ -188,6 +218,34 @@ class HitchRetryingVerifier(Verifier):
 
         raise AssertionError("unreachable verifier retry state")
 
+    def _read_trusted_agent_outcome(self) -> dict[str, Any] | None:
+        source = self.trial_paths.verifier_dir.parent / "agent" / HITCH_AGENT_OUTCOME_NAME
+        try:
+            if not source.is_file() or source.stat().st_size <= 0 or source.stat().st_size > 16 * 1024:
+                return None
+            value = json.loads(source.read_text(encoding="utf-8"))
+        except (OSError, UnicodeError, json.JSONDecodeError):
+            return None
+        if not isinstance(value, dict) or set(value) - {
+            "schema_version", "run_id", "status", "candidate_bundle", "submission_snapshot", "gradeability", "reason_code",
+        }:
+            return None
+        if value.get("schema_version") != "1" or re.fullmatch(r"run_[a-f0-9]{32}", str(value.get("run_id", ""))) is None:
+            return None
+        if value.get("status") not in {"succeeded", "failed", "timed_out", "cancelled"}:
+            return None
+        if value.get("candidate_bundle") not in {"complete", "missing", "invalid"}:
+            return None
+        if value.get("submission_snapshot") not in {"complete", "missing", "not-required"}:
+            return None
+        if value.get("gradeability") not in {"gradeable", "ungradeable"}:
+            return None
+        if value["gradeability"] == "gradeable" and value["candidate_bundle"] != "complete":
+            return None
+        if "reason_code" in value and (not isinstance(value["reason_code"], str) or not value["reason_code"]):
+            return None
+        return value
+
     def _write_phase_timing(self, started_ns: int) -> None:
         self._write_json(
             "hitch-phase-timings.json",