agent-hitch 0.2.6 → 0.2.8

package/integrations/harbor/hitch_harbor_task_resources.py

@@ -0,0 +1,275 @@
+"""Inspect Harbor task and Compose resource declarations without starting Docker."""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+import tomllib
+from decimal import Decimal, InvalidOperation
+from pathlib import Path
+from typing import Any
+
+import yaml
+from harbor.models.task.config import TaskConfig, VerifierEnvironmentMode
+
+
+def inspect_task(task_dir: Path) -> dict[str, Any]:
+    task_file = task_dir / "task.toml"
+    if not task_file.is_file():
+        raise ValueError(f"task.toml is missing: {task_file}")
+    with task_file.open("rb") as handle:
+        config = TaskConfig.model_validate(tomllib.load(handle))
+    services = _compose_services(task_dir / "environment" / "docker-compose.yaml")
+    runtime_platform = _compose_runtime_platform(
+        task_dir / "environment" / "docker-compose.yaml"
+    )
+    if "main" not in {service["name"] for service in services}:
+        services.append({"name": "main", "replicas": 1})
+    services.sort(key=lambda service: service["name"].encode("utf-8"))
+
+    verifier_mode = config.verifier.environment_mode
+    separate = verifier_mode == VerifierEnvironmentMode.SEPARATE or (
+        verifier_mode is None and config.verifier.environment is not None
+    )
+    verifier_environment = config.verifier.environment
+    if separate and verifier_environment is None:
+        verifier_environment = config.environment
+    main_egress = config.environment.os.value == "linux" and any(
+        mode is not None and mode.value != "public"
+        for mode in (
+            config.environment.network_mode,
+            config.agent.network_mode,
+            None if separate else config.verifier.network_mode,
+        )
+    )
+    verifier_egress = bool(
+        separate
+        and verifier_environment is not None
+        and verifier_environment.os.value == "linux"
+        and any(
+            mode is not None and mode.value != "public"
+            for mode in (verifier_environment.network_mode, config.verifier.network_mode)
+        )
+    )
+    images, fallbacks, builds = _environment_images(
+        config.environment, "task", task_dir / "environment"
+    )
+    if separate and verifier_environment is not None:
+        verifier_images, verifier_fallbacks, verifier_builds = _environment_images(
+            verifier_environment, "verifier", None
+        )
+        images.extend(verifier_images)
+        fallbacks.extend(verifier_fallbacks)
+        builds.extend(verifier_builds)
+    compose_images, compose_fallbacks = _compose_images(
+        task_dir / "environment" / "docker-compose.yaml"
+    )
+    images.extend(compose_images)
+    fallbacks.extend(compose_fallbacks)
+    return {
+        "schema_version": "1",
+        **({"runtime_platform": runtime_platform} if runtime_platform is not None else {}),
+        "task": _environment_resources(config.environment),
+        "verifier": {
+            "separate": separate,
+            **({"environment": _environment_resources(verifier_environment)} if verifier_environment is not None else {}),
+        },
+        "compose_services": services,
+        "provider_sidecars": {
+            "main_egress": main_egress,
+            "verifier_egress": verifier_egress,
+        },
+        "environment_images": sorted(
+            images, key=lambda entry: (entry["source"], entry["service"])
+        ),
+        "environment_image_fallbacks": sorted(
+            fallbacks, key=lambda entry: (entry["source"], entry["service"])
+        ),
+        "environment_builds": sorted(
+            builds, key=lambda entry: (entry["source"], entry["service"])
+        ),
+    }
+
+
+def _compose_runtime_platform(compose_file: Path) -> str | None:
+    if not compose_file.exists():
+        return None
+    document = yaml.safe_load(compose_file.read_text(encoding="utf-8"))
+    if not isinstance(document, dict):
+        return None
+    services = document.get("services")
+    main = services.get("main") if isinstance(services, dict) else None
+    raw = main.get("platform") if isinstance(main, dict) else None
+    if raw is None:
+        return None
+    if not isinstance(raw, str):
+        raise ValueError("Compose main platform must be a string")
+    normalized = {
+        "linux/x86_64": "linux/amd64",
+        "linux/aarch64": "linux/arm64",
+    }.get(raw.strip().lower(), raw.strip().lower())
+    if normalized not in {"linux/amd64", "linux/arm64"}:
+        raise ValueError(f"Compose main platform is unsupported: {raw}")
+    return normalized
+
+
+def _environment_resources(environment: Any) -> dict[str, Any]:
+    return {
+        **({"cpu_millis": environment.cpus * 1000} if environment.cpus is not None else {}),
+        **({"memory_bytes": environment.memory_mb * 1024 * 1024} if environment.memory_mb is not None else {}),
+    }
+
+
+def _environment_images(
+    environment: Any, source: str, context_dir: Path | None
+) -> tuple[list[dict[str, str]], list[dict[str, str]], list[dict[str, str]]]:
+    reference = getattr(environment, "docker_image", None)
+    if reference is None:
+        if context_dir is not None and (context_dir / "Dockerfile").is_file():
+            return [], [], [{
+                "source": source,
+                "service": "main",
+                "context": "environment",
+                "dockerfile": "Dockerfile",
+            }]
+        return [], [{"source": source, "service": "main", "code": "backend-build"}], []
+    if not isinstance(reference, str) or not reference or "$" in reference:
+        return [], [{"source": source, "service": "main", "code": "dynamic-image"}], []
+    return [{"source": source, "service": "main", "reference": reference}], [], []
+
+
+def _compose_images(
+    compose_file: Path,
+) -> tuple[list[dict[str, str]], list[dict[str, str]]]:
+    if not compose_file.exists():
+        return [], []
+    document = yaml.safe_load(compose_file.read_text(encoding="utf-8"))
+    if document is None or not isinstance(document, dict):
+        return [], []
+    raw_services = document.get("services")
+    if not isinstance(raw_services, dict):
+        return [], []
+    images: list[dict[str, str]] = []
+    fallbacks: list[dict[str, str]] = []
+    for name, raw in raw_services.items():
+        if not isinstance(name, str) or not isinstance(raw, dict):
+            continue
+        reference = raw.get("image")
+        if raw.get("build") is not None:
+            fallbacks.append({"source": "compose", "service": name, "code": "backend-build"})
+        elif isinstance(reference, str) and reference and "$" not in reference:
+            images.append({"source": "compose", "service": name, "reference": reference})
+        elif reference is not None:
+            fallbacks.append({"source": "compose", "service": name, "code": "dynamic-image"})
+    return images, fallbacks
+
+
+def _compose_services(compose_file: Path) -> list[dict[str, Any]]:
+    if not compose_file.exists():
+        return []
+    document = yaml.safe_load(compose_file.read_text(encoding="utf-8"))
+    if document is None:
+        return []
+    if not isinstance(document, dict) or "include" in document:
+        raise ValueError("Compose resource discovery requires one mapping without include")
+    raw_services = document.get("services")
+    if not isinstance(raw_services, dict):
+        raise ValueError("Compose services must be a mapping")
+    result: list[dict[str, Any]] = []
+    for name, raw in raw_services.items():
+        if not isinstance(name, str) or not name or not isinstance(raw, dict):
+            raise ValueError("Compose service declaration is invalid")
+        if raw.get("profiles") or raw.get("extends"):
+            raise ValueError(f"Compose resource discovery does not support profiles/extends: {name}")
+        deploy = raw.get("deploy") if isinstance(raw.get("deploy"), dict) else {}
+        deploy_resources = deploy.get("resources") if isinstance(deploy.get("resources"), dict) else {}
+        limits = deploy_resources.get("limits") if isinstance(deploy_resources.get("limits"), dict) else {}
+        reservations = deploy_resources.get("reservations") if isinstance(deploy_resources.get("reservations"), dict) else {}
+        cpu_values = [_cpu_millis(value, name) for value in (raw.get("cpus"), limits.get("cpus")) if value is not None]
+        memory_values = [_memory_bytes(value, name) for value in (raw.get("mem_limit"), limits.get("memory")) if value is not None]
+        gpu_values = [_gpu_count(value, name) for value in (raw.get("gpus"), reservations.get("devices")) if value is not None]
+        replicas = raw.get("scale", deploy.get("replicas", 1))
+        if isinstance(replicas, bool) or not isinstance(replicas, int) or replicas < 1:
+            raise ValueError(f"Compose service replicas must be a positive integer: {name}")
+        result.append({
+            "name": name,
+            "replicas": replicas,
+            **({"cpu_millis": max(cpu_values)} if cpu_values else {}),
+            **({"memory_bytes": max(memory_values)} if memory_values else {}),
+            **({"gpu_count": max(gpu_values)} if gpu_values else {}),
+        })
+    return result
+
+
+def _gpu_count(value: Any, service: str) -> int:
+    if value == "all":
+        raise ValueError(f"Compose GPU request must use a fixed count: {service}")
+    if not isinstance(value, list) or not value:
+        raise ValueError(f"Compose GPU request is invalid: {service}")
+    total = 0
+    for request in value:
+        if not isinstance(request, dict):
+            raise ValueError(f"Compose GPU request is invalid: {service}")
+        capabilities = request.get("capabilities")
+        if not isinstance(capabilities, list) or "gpu" not in capabilities:
+            continue
+        count = request.get("count")
+        device_ids = request.get("device_ids")
+        if count is not None and device_ids is not None:
+            raise ValueError(f"Compose GPU request cannot set count and device_ids: {service}")
+        if device_ids is not None:
+            if not isinstance(device_ids, list) or not device_ids or any(not isinstance(item, str) or not item for item in device_ids):
+                raise ValueError(f"Compose GPU device_ids are invalid: {service}")
+            total += len(device_ids)
+        elif isinstance(count, int) and not isinstance(count, bool) and count > 0:
+            total += count
+        else:
+            raise ValueError(f"Compose GPU request must use a fixed count: {service}")
+    if total < 1:
+        raise ValueError(f"Compose GPU request has no gpu capability: {service}")
+    return total
+
+
+def _cpu_millis(value: Any, service: str) -> int:
+    if isinstance(value, bool) or not isinstance(value, (int, float, str)):
+        raise ValueError(f"Compose CPU limit is not numeric: {service}")
+    try:
+        decimal = Decimal(str(value).strip()) * 1000
+    except InvalidOperation as error:
+        raise ValueError(f"Compose CPU limit is not numeric: {service}") from error
+    if decimal != decimal.to_integral_value() or decimal <= 0:
+        raise ValueError(f"Compose CPU limit must be positive whole millicores: {service}")
+    return int(decimal)
+
+
+def _memory_bytes(value: Any, service: str) -> int:
+    if isinstance(value, bool):
+        raise ValueError(f"Compose memory limit is invalid: {service}")
+    if isinstance(value, int):
+        if value <= 0:
+            raise ValueError(f"Compose memory limit must be positive: {service}")
+        return value
+    if not isinstance(value, str):
+        raise ValueError(f"Compose memory limit is invalid: {service}")
+    match = re.fullmatch(r"\s*([0-9]+(?:\.[0-9]+)?)\s*([kmgtpe]?i?b?)?\s*", value, re.IGNORECASE)
+    if not match:
+        raise ValueError(f"Compose memory limit is not a fixed size: {service}")
+    unit = (match.group(2) or "b").lower()
+    powers = {"b": 0, "k": 1, "kb": 1, "ki": 1, "kib": 1, "m": 2, "mb": 2, "mi": 2, "mib": 2,
+              "g": 3, "gb": 3, "gi": 3, "gib": 3, "t": 4, "tb": 4, "ti": 4, "tib": 4,
+              "p": 5, "pb": 5, "pi": 5, "pib": 5, "e": 6, "eb": 6, "ei": 6, "eib": 6}
+    bytes_value = Decimal(match.group(1)) * (1024 ** powers[unit])
+    if bytes_value != bytes_value.to_integral_value() or bytes_value <= 0:
+        raise ValueError(f"Compose memory limit is not a whole positive byte size: {service}")
+    return int(bytes_value)
+
+
+def main() -> None:
+    if len(sys.argv) != 2:
+        raise SystemExit("usage: hitch_harbor_task_resources.py <task-directory>")
+    print(json.dumps(inspect_task(Path(sys.argv[1]).resolve()), separators=(",", ":"), sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()

package/integrations/harbor/hitch_harbor_verifier.py

@@ -11,7 +11,9 @@ import asyncio
 import json
 import re
 import shutil
+import time
 from dataclasses import dataclass
+from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any
 
@@ -21,9 +23,15 @@ from harbor.verifier.verifier import Verifier
 
 
 MAX_LOG_BYTES = 256 * 1024
+HITCH_AGENT_OUTCOME_NAME = "hitch-agent-outcome.json"
 LOG_NAMES = ("test-stdout.txt", "test-stderr.txt", "stdout.txt", "stderr.txt")
-OUTPUT_NAMES = (*LOG_NAMES, "reward.txt", "reward.json", "ctrf.json")
-CONTROL_NAMES = ("infrastructure-error.json", "infrastructure-retry-history.json")
+OUTPUT_NAMES = (*LOG_NAMES, "reward.txt", "reward.json", "process.json", "feedback.json", "ctrf.json")
+CONTROL_NAMES = (
+    "infrastructure-error.json",
+    "infrastructure-retry-history.json",
+    "candidate-ineligible.json",
+    "eligibility-gate.json",
+)
 
 INFRASTRUCTURE_PATTERNS: tuple[tuple[str, tuple[re.Pattern[str], ...]], ...] = (
     (
@@ -117,8 +125,35 @@ class HitchRetryingVerifier(Verifier):
         self.infrastructure_retry_backoff_ms = infrastructure_retry_backoff_ms
 
     async def verify(self) -> VerifierResult:
+        started_ns = time.monotonic_ns()
+        try:
+            return await self._verify_with_retries()
+        finally:
+            self._write_phase_timing(started_ns)
+
+    async def _verify_with_retries(self) -> VerifierResult:
+        directory = getattr(getattr(getattr(self, "task", None), "paths", None), "environment_dir", None)
+        if directory and (directory.parent / ".hitch-benchmark.json").is_file():
+            from hitch_benchmark import restore_final_response, validate_collected_submission
+            validate_collected_submission(self)
+            await restore_final_response(self)
         # Do not trust a control file left by the candidate or a prior phase.
         await self._remove_files(CONTROL_NAMES)
+        eligibility = self._read_trusted_agent_outcome()
+        if eligibility is None:
+            self._write_json("eligibility-gate.json", {
+                "schema_version": "1", "status": "unavailable", "verifier_executed": True,
+            })
+        elif eligibility["gradeability"] == "ungradeable":
+            self._write_json("candidate-ineligible.json", {
+                "schema_version": "1",
+                "code": "candidate_evidence_unavailable",
+                "run_id": eligibility["run_id"],
+                "candidate_bundle": eligibility["candidate_bundle"],
+                "reason_code": eligibility.get("reason_code", "candidate_evidence_unavailable"),
+                "verifier_executed": False,
+            })
+            return VerifierResult(rewards={"reward": 0})
         attempts: list[dict[str, Any]] = []
         for attempt in range(1, self.infrastructure_retries + 2):
             result: VerifierResult | None = None
@@ -143,6 +178,10 @@ class HitchRetryingVerifier(Verifier):
                 if attempts:
                     self._write_history("recovered", attempts)
                 assert result is not None
+                directory = getattr(getattr(getattr(self, "task", None), "paths", None), "environment_dir", None)
+                if directory and (directory.parent / ".hitch-benchmark.json").is_file():
+                    from hitch_benchmark import normalize_rewards
+                    return normalize_rewards(self, result)
                 return result
 
             attempts.append(
@@ -179,6 +218,48 @@ class HitchRetryingVerifier(Verifier):
 
         raise AssertionError("unreachable verifier retry state")
 
+    def _read_trusted_agent_outcome(self) -> dict[str, Any] | None:
+        source = self.trial_paths.verifier_dir.parent / "agent" / HITCH_AGENT_OUTCOME_NAME
+        try:
+            if not source.is_file() or source.stat().st_size <= 0 or source.stat().st_size > 16 * 1024:
+                return None
+            value = json.loads(source.read_text(encoding="utf-8"))
+        except (OSError, UnicodeError, json.JSONDecodeError):
+            return None
+        if not isinstance(value, dict) or set(value) - {
+            "schema_version", "run_id", "status", "candidate_bundle", "submission_snapshot", "gradeability", "reason_code",
+        }:
+            return None
+        if value.get("schema_version") != "1" or re.fullmatch(r"run_[a-f0-9]{32}", str(value.get("run_id", ""))) is None:
+            return None
+        if value.get("status") not in {"succeeded", "failed", "timed_out", "cancelled"}:
+            return None
+        if value.get("candidate_bundle") not in {"complete", "missing", "invalid"}:
+            return None
+        if value.get("submission_snapshot") not in {"complete", "missing", "not-required"}:
+            return None
+        if value.get("gradeability") not in {"gradeable", "ungradeable"}:
+            return None
+        if value["gradeability"] == "gradeable" and value["candidate_bundle"] != "complete":
+            return None
+        if "reason_code" in value and (not isinstance(value["reason_code"], str) or not value["reason_code"]):
+            return None
+        return value
+
+    def _write_phase_timing(self, started_ns: int) -> None:
+        self._write_json(
+            "hitch-phase-timings.json",
+            {
+                "schema_version": "1",
+                "phases": {
+                    "verifier": {
+                        "duration_ms": max(0, (time.monotonic_ns() - started_ns) // 1_000_000),
+                        "completed_at": datetime.now(timezone.utc).isoformat(),
+                    }
+                },
+            },
+        )
+
     async def _clear_outputs(self) -> None:
         await self._remove_files(OUTPUT_NAMES)