agent-hitch 0.2.16 → 0.2.17

package/integrations/harbor/hitch_harbor_agent.py

@@ -128,7 +128,6 @@ class HitchHarborAgent(BaseAgent):
         verifier_identity: str | None = None,
         logical_attempt: int | None = None,
         model_capture: dict[str, Any] | None = None,
-        managed_local_inference: dict[str, Any] | None = None,
         **kwargs: Any,
     ) -> None:
         super().__init__(logs_dir=logs_dir, **kwargs)
@@ -167,14 +166,6 @@ class HitchHarborAgent(BaseAgent):
             raise ValueError("logical_attempt must be a positive integer")
         self.logical_attempt = logical_attempt
         self.model_capture = _validate_model_capture(model_capture)
-        if managed_local_inference is not None and not _valid_managed_model_identity(managed_local_inference):
-            raise ValueError("managed_local_inference identity is invalid")
-        if managed_local_inference is not None and self.model_capture is None:
-            raise ValueError("managed_local_inference requires model_capture")
-        self.managed_local_inference = dict(managed_local_inference) if managed_local_inference else None
-        capture_inference = self.model_capture.get("managed_inference") if self.model_capture else None
-        if capture_inference != self.managed_local_inference:
-            raise ValueError("managed_local_inference must match model_capture identity")
         self._hitch_version: str | None = None
         self._entrypoint: str | None = None
         self._artifact_manifest: dict[str, Any] | None = None
@@ -249,7 +240,10 @@ class HitchHarborAgent(BaseAgent):
         # Upload the cached bundle's payload (package.json + dist/) as the
         # package root under /opt/hitch; the local cache path is host-side
         # bookkeeping and is not identity (spec §4.2).
-        await environment.upload_dir(payload_dir, "/opt/hitch")
+        shared = getattr(environment, "hitch_verify_shared_runtime", None)
+        shared_ready = await shared(payload_dir, self._artifact_host_directory) if shared else False
+        if not shared_ready:
+            await environment.upload_dir(payload_dir, "/opt/hitch")
         await self._ensure_node(environment)
         platform, node_version = await self._container_node_identity(environment)
         if self._artifact_manifest is None or not self._artifact_compatible(
@@ -259,8 +253,12 @@ class HitchHarborAgent(BaseAgent):
                 "hitch-artifact-platform: dedicated-builder artifact is incompatible with "
                 f"trial runtime {platform}/{node_version}"
             )
-        await self._upload_harness_artifact(environment, self._artifact_host_directory)
-        self._artifact_transport_status = "dedicated_builder_upload"
+        if shared_ready:
+            self._artifact_uploaded = True
+            self._artifact_transport_status = "shared_readonly_bind"
+        else:
+            await self._upload_harness_artifact(environment, self._artifact_host_directory)
+            self._artifact_transport_status = "dedicated_builder_upload"
         entry = self._remote_entry(entrypoint)
         version = await self._exec(environment, f"{self._node_prefix()} node {entry} --version")
         self._hitch_version = (version.stdout or "").strip() or None
@@ -1417,6 +1415,7 @@ mv "$stage_dir" "$target_dir"
             return [], "degraded-unreachable"
         base = self.model_capture["base_url_template"].replace("{run_id}", run_id)
         training = self.model_capture.get("training_external")
+        external = self.model_capture.get("external_endpoint")
         return [
             f"OPENAI_BASE_URL={shlex.quote(base.replace('{provider}', 'openai'))}",
             f"ANTHROPIC_BASE_URL={shlex.quote(base.replace('{provider}', 'anthropic'))}",
@@ -1427,14 +1426,10 @@ mv "$stage_dir" "$target_dir"
                 "OPENAI_API_KEY=hitch-training-external",
             ] if training else []),
             *([
-                "HITCH_MANAGED_LOCAL_INFERENCE=1",
-                f"HITCH_MANAGED_RUN_ID={run_id}",
-                f"HITCH_MANAGED_INFERENCE_ID={self.managed_local_inference['inference_id']}",
-                f"HITCH_MANAGED_MODEL_ID={self.managed_local_inference['model_id']}",
-                "OPENAI_API_KEY=hitch-managed-local",
-                *([f"HITCH_MANAGED_NODE_BINDING={shlex.quote(json.dumps(self.managed_local_inference['model_node'], separators=(',', ':')))}"]
-                  if self.managed_local_inference.get("model_node") else []),
-            ] if self.managed_local_inference else []),
+                f"HITCH_EXTERNAL_RUN_ID={run_id}",
+                f"HITCH_EXTERNAL_ENDPOINT_BINDING={shlex.quote(json.dumps(external, separators=(',', ':')))}",
+                "OPENAI_API_KEY=hitch-external-endpoint",
+            ] if external else []),
         ], "healthy"
 
     @staticmethod
@@ -1899,7 +1894,7 @@ def _validate_model_capture(value: dict[str, Any] | None) -> dict[str, Any] | No
     required_fields = {
         "schema_version", "mode", "required", "topology", "base_url_template", "health_url_template"
     }
-    if not isinstance(value, dict) or not required_fields.issubset(value) or set(value) - required_fields - {"managed_inference", "training_external"}:
+    if not isinstance(value, dict) or not required_fields.issubset(value) or set(value) - required_fields - {"external_endpoint", "training_external"}:
         raise ValueError("model_capture fields are invalid")
     if (
         value.get("schema_version") != "1"
@@ -1924,35 +1919,26 @@ def _validate_model_capture(value: dict[str, Any] | None) -> dict[str, Any] | No
         )
         if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.username or parsed.password:
             raise ValueError(f"model_capture {field} URL is invalid")
-    managed = value.get("managed_inference")
+    external = value.get("external_endpoint")
     training = value.get("training_external")
     if training is not None:
-        if (managed is not None or not isinstance(training, dict)
+        if (external is not None or not isinstance(training, dict)
             or set(training) != {"binding_id", "training_run_id", "policy_version", "generation_contract_digest", "api", "max_output_tokens", "max_episode_steps"}
             or training.get("api") != "chat-completions" or value.get("required") is not True
             or any(not isinstance(training.get(k), str) or not training[k] for k in ("binding_id", "training_run_id", "policy_version"))
             or re.fullmatch(r"sha256:[a-f0-9]{64}", str(training.get("generation_contract_digest"))) is None
             or any(type(training.get(k)) is not int or training[k] <= 0 for k in ("max_output_tokens", "max_episode_steps"))):
             raise ValueError("model_capture training identity is invalid")
-    if managed is not None and not _valid_managed_model_identity(managed):
-        raise ValueError("model_capture managed inference identity is invalid")
+    if external is not None:
+        if (not isinstance(external, dict) or external.get("kind") != "external-endpoint"
+            or external.get("schema_version") != "1" or "connection" in external
+            or external.get("api") not in {"responses", "chat-completions"}
+            or re.fullmatch(r"sha256:[a-f0-9]{64}", str(external.get("endpoint_revision"))) is None
+            or value.get("required") is not True):
+            raise ValueError("model_capture external endpoint binding is invalid")
     return dict(value)
 
 
-def _valid_managed_model_identity(value: Any) -> bool:
-    if (not isinstance(value, dict) or not {"inference_id", "model_id"}.issubset(value)
-        or set(value) - {"inference_id", "model_id", "model_node"}
-        or any(not isinstance(value[k], str) or re.fullmatch(r"sha256:[a-f0-9]{64}", value[k]) is None for k in ("inference_id", "model_id"))):
-        return False
-    if "model_node" not in value:
-        return True
-    node = value["model_node"]
-    return (isinstance(node, dict) and set(node) == {"schema_version", "node_id", "generation", "runtime_digest", "launcher"}
-        and node["schema_version"] == "2" and node["launcher"] == "process"
-        and all(isinstance(node[k], str) and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", node[k]) is not None for k in ("node_id", "generation"))
-        and isinstance(node["runtime_digest"], str) and re.fullmatch(r"sha256:[a-f0-9]{64}", node["runtime_digest"]) is not None)
-
-
 def canonical_manifest_json(manifest: dict[str, Any]) -> str:
     """Canonically encode the runtime identity `{ schema_version, node_range,
     entrypoints, files }` with sorted object keys and no insignificant

package/integrations/harbor/hitch_harbor_environment.py

@@ -34,10 +34,24 @@ _GPU_OVERRIDE_PREFIX = "__HITCH_GPU_OVERRIDE_"
 _HOST_CREDENTIAL_HELPER_ENV = "HITCH_HOST_CREDENTIAL_HELPER_JSON"
 
 
+class _SharedRuntimeConfigurationError(ValueError):
+    """No candidate was started; cleanup must preserve existing images."""
+
+
+def _candidate_session(session_id: str, trial_name: str | None) -> bool:
+    # Keep role selection self-contained: historical verifier repairs replace
+    # only this provider and retain the original runtime's module membership.
+    # Verifier sessions may truncate their role marker or use a step named env.
+    if trial_name is not None:
+        return session_id == f"{trial_name}__env"
+    return session_id.endswith("__env") and "__verifier__" not in session_id
+
+
 class HitchHarborDockerEnvironment(DockerEnvironment):
     """Use Harbor's Docker semantics with a final ownership-label overlay."""
 
     _hitch_static_no_network = False
+    _hitch_preparation = None
 
     def __init__(
         self,
@@ -48,6 +62,10 @@ class HitchHarborDockerEnvironment(DockerEnvironment):
         hitch_prebuilt_task_image: str | None = None,
         hitch_model_proxy_host_gateway: bool = False,
         hitch_main_gpu_count: int = 0,
+        hitch_shared_runtime: Mapping[str, str] | None = None,
+        hitch_image_cache_dir: str | None = None,
+        hitch_image_build_slots: int = 2,
+        hitch_managed_keepalive: bool = False,
         **kwargs: Any,
     ) -> None:
         self._hitch_ownership_labels = _validate_labels(hitch_ownership_labels)
@@ -58,12 +76,32 @@ class HitchHarborDockerEnvironment(DockerEnvironment):
         self._hitch_prebuilt_task_image = _validate_prebuilt_task_image(
             hitch_prebuilt_task_image
         )
+        # Harbor passes environment kwargs to separate verifiers too. A planned
+        # candidate image must never replace a verifier's independent context.
+        session_id = kwargs.get("session_id", args[2] if len(args) > 2 else "")
+        trial_paths = kwargs.get("trial_paths", args[3] if len(args) > 3 else None)
+        trial_name = trial_paths.trial_dir.name if trial_paths is not None else None
+        candidate = _candidate_session(session_id, trial_name)
+        if session_id and not candidate:
+            self._hitch_prebuilt_task_image = None
+        if hitch_image_cache_dir is not None and (not isinstance(hitch_image_cache_dir, str) or not Path(hitch_image_cache_dir).is_absolute()):
+            raise ValueError("Hitch image cache directory must be absolute")
+        if not isinstance(hitch_managed_keepalive, bool):
+            raise ValueError("Hitch managed keepalive flag is invalid")
+        self._hitch_preparation = None
+        if hitch_image_cache_dir or hitch_managed_keepalive:
+            from hitch_compose_preparation import ComposePreparation
+            self._hitch_preparation = ComposePreparation(self, hitch_image_cache_dir, hitch_image_build_slots, hitch_managed_keepalive)
         if not isinstance(hitch_model_proxy_host_gateway, bool):
             raise ValueError("Hitch model proxy host gateway flag is invalid")
         self._hitch_model_proxy_host_gateway = hitch_model_proxy_host_gateway
         if isinstance(hitch_main_gpu_count, bool) or not isinstance(hitch_main_gpu_count, int) or hitch_main_gpu_count < 0:
             raise ValueError("Hitch main GPU count is invalid")
         self._hitch_main_gpu_count = hitch_main_gpu_count
+        self._hitch_shared_runtime = None
+        if hitch_shared_runtime is not None:
+            import hitch_shared_runtime as shared_runtime
+            self._hitch_shared_runtime = shared_runtime.configure(hitch_shared_runtime, candidate)
         self._hitch_ownership_temp_dir: tempfile.TemporaryDirectory[str] | None = None
         self._hitch_ownership_compose_path: Path | None = None
         self._hitch_phase_compose_path: Path | None = None
@@ -103,6 +141,7 @@ class HitchHarborDockerEnvironment(DockerEnvironment):
             or self._hitch_main_gpu_count > 0
             or self._hitch_benchmark_platform
             or self._hitch_static_no_network
+            or self._hitch_shared_runtime
         ):
             self._hitch_ownership_compose_path = self._write_ownership_overlay()
 
@@ -113,10 +152,52 @@ class HitchHarborDockerEnvironment(DockerEnvironment):
         paths = list(super()._docker_compose_paths)
         if self._hitch_ownership_compose_path is not None:
             paths.append(self._hitch_ownership_compose_path)
+        if self._hitch_preparation and self._hitch_preparation.path is not None:
+            paths.append(self._hitch_preparation.path)
         if self._hitch_phase_compose_path is not None:
             paths.append(self._hitch_phase_compose_path)
         return paths
 
+    @property
+    def _main_image_name(self):
+        preparation = getattr(self, "_hitch_preparation", None)
+        return preparation.images.get(MAIN_SERVICE_NAME, super()._main_image_name) if preparation else super()._main_image_name
+
+    async def _run_docker_compose_command(self, command, *args, **kwargs):
+        run = super()._run_docker_compose_command
+        if command[:1] == ["up"] and self._hitch_shared_runtime:
+            import hitch_shared_runtime as shared_runtime
+            from hitch_compose_preparation import compose_services
+            # Resolve the task's merged/interpolated configuration before our
+            # bind mounts can hide a conflicting task mount. Revalidate on every
+            # start, and never expose host snapshots to a privileged candidate.
+            try:
+                self._hitch_ownership_compose_path = self._write_ownership_overlay()
+                resolved = await run(["config", "--format", "json"])
+                shared_runtime.reject_conflicts(compose_services(resolved.stdout))
+                self._hitch_ownership_compose_path = self._write_ownership_overlay(include_shared_runtime=True)
+            except Exception as error:
+                raise _SharedRuntimeConfigurationError(str(error)) from error
+        preparation = self._hitch_preparation
+        if preparation:
+            if command == ["build"]:
+                cached = await preparation.build(run)
+                if cached is not None:
+                    return cached
+            elif command[:1] == ["up"]:
+                # If Harbor selected a prebuilt image and skipped build, retain
+                # Compose's original image/pull-policy choices, including any
+                # sidecars. An implicit up is not an explicit build request.
+                await preparation.runtime(run)
+            elif command[:1] == ["down"] and preparation.images and "--rmi" in command:
+                index = command.index("--rmi")
+                if command[index + 1:index + 2] == ["local"]:
+                    # Compose treats an immutable image ID as an untagged local
+                    # image even though our persistent cache also owns a tag.
+                    # Keep prepared images; retain all container/volume cleanup.
+                    command = command[:index] + command[index + 2:]
+        return await run(command, *args, **kwargs)
+
     def _requires_egress_control(self, *, startup_network_policy, phase_network_policies):
         if self._hitch_static_no_network:
             return False
@@ -148,6 +229,12 @@ class HitchHarborDockerEnvironment(DockerEnvironment):
             user=user,
         )
 
+    async def hitch_verify_shared_runtime(self, controller_payload, artifact_directory):
+        if self._hitch_shared_runtime is None:
+            return False
+        import hitch_shared_runtime as shared_runtime
+        return await shared_runtime.verify_environment(self, controller_payload, artifact_directory)
+
     async def _apply_network_policy(self, network_policy):
         if self._hitch_static_no_network:
             if network_policy != self.network_policy:
@@ -188,15 +275,17 @@ class HitchHarborDockerEnvironment(DockerEnvironment):
 
     async def start(self, force_build: bool):
         try:
+            if self._hitch_preparation:
+                self._hitch_preparation.reset(force_build)
             await super().start(force_build)
             if self._hitch_static_no_network:
                 await self._verify_static_network()
             if self._hitch_benchmark:
                 await self._hitch_benchmark.prepare()
-        except BaseException:
+        except BaseException as error:
             # Harbor may abort setup before entering its normal teardown path.
             try:
-                await self.stop(delete=True)
+                await self.stop(delete=not isinstance(error, _SharedRuntimeConfigurationError))
             except BaseException:
                 pass
             raise
@@ -222,7 +311,9 @@ class HitchHarborDockerEnvironment(DockerEnvironment):
         finally:
             await super().stop(delete)
 
-    def _write_ownership_overlay(self) -> Path:
+    def _write_ownership_overlay(self, *, include_shared_runtime=False) -> Path:
+        if include_shared_runtime and self._hitch_shared_runtime:
+            import hitch_shared_runtime as shared_runtime
         services = {MAIN_SERVICE_NAME}
         if self._enable_egress_control:
             services.add(self._EGRESS_CONTROL_SERVICE_NAME)
@@ -265,6 +356,8 @@ class HitchHarborDockerEnvironment(DockerEnvironment):
         service_overlays: dict[str, dict[str, Any]] = {}
         for name in sorted(services):
             config: dict[str, Any] = {}
+            if name == MAIN_SERVICE_NAME and include_shared_runtime and self._hitch_shared_runtime:
+                config["volumes"] = shared_runtime.compose_mounts(self._hitch_shared_runtime)
             # The benchmark default must not replace explicit per-service
             # architecture, e.g. an ARM QEMU sidecar beside an AMD64 candidate.
             if getattr(self, "_hitch_benchmark_platform", None) and name not in declared_platforms: