agent-hitch 0.2.6 → 0.2.8

package/integrations/harbor/hitch_benchmark.py

@@ -0,0 +1,252 @@
+"""Generic benchmark package hooks, tool binding and metric normalization.
+
+Runs in the trusted Harbor worker. Package business code is never imported here.
+"""
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+import math
+from pathlib import Path
+import shlex
+import tempfile
+import uuid
+
+
+def descriptor(environment_dir):
+    path = Path(environment_dir).parent / ".hitch-benchmark.json"
+    return json.loads(path.read_text()) if path.is_file() else None
+
+
+class BenchmarkSession:
+    def __init__(self, env, config):
+        self.env, self.config = env, config
+        self.responses = {}
+        self.started = False
+        self.stopped = False
+        self.failure = None
+        self.journal = env.trial_paths.trial_dir / "benchmark-lifecycle.json"
+
+    async def phase(self, phase):
+        if phase in self.responses:
+            return self.responses[phase]
+        hook = self.config["task"]["lifecycle"][phase]
+        labels = self.env._hitch_ownership_labels
+        request = {
+            "schema_version": "1",
+            "request_id": str(uuid.uuid5(uuid.NAMESPACE_URL, self.env.session_id + ":" + phase)),
+            "phase": phase, "task_id": self.config["task_id"],
+            "logical_trial_id": self.env.session_id, "execution_index": 0,
+            "lease_id": labels.get("io.hitch.lease-id", self.env.session_id),
+            "epoch": int(labels.get("io.hitch.lease-epoch", "1")),
+            "profile_digest": self.config["profile_digest"], "input_refs": [],
+        }
+        # Harbor's exec API accepts shell text. Quote each argv and stdin byte;
+        # no task text is interpolated as shell syntax.
+        command = "printf %s " + shlex.quote(json.dumps(request)) + " | " + shlex.join(hook["argv"])
+        try:
+            result = await self.env.service_exec(command, service=hook["target"].split(":", 1)[1], timeout_sec=math.ceil(hook["timeout_ms"] / 1000))
+            if result.return_code != 0:
+                raise RuntimeError(f"hook {phase} exited {result.return_code}: {result.stderr}")
+            if len(result.stdout or "") > 1024 * 1024:
+                raise RuntimeError("oversized hook response")
+            response = json.loads(result.stdout)
+            if set(response) != {"schema_version", "request_id", "status", "output"}:
+                raise RuntimeError(f"invalid hook response fields: {phase}")
+            if response.get("schema_version") != "1" or response.get("request_id") != request["request_id"] or response.get("status") != "ok":
+                raise RuntimeError(f"invalid/failed hook response: {phase}")
+            output = response["output"]
+            required = {"prepare": "ready", "quiesce": "quiesced", "cleanup": "cleaned"}.get(phase)
+            if required and output.get(required) is not True:
+                raise RuntimeError(f"hook {phase} did not confirm {required}")
+            if phase == "snapshot":
+                paths = self.config["task"]["submission"]["paths"]
+                artifacts = output.get("artifacts", [])
+                if not isinstance(artifacts, list) or any(type(a.get("bytes")) is not int or a["bytes"] < 0 for a in artifacts):
+                    raise RuntimeError("invalid snapshot size metadata")
+                if sorted(a.get("path", "") for a in artifacts) != sorted(paths):
+                    raise RuntimeError("snapshot artifact membership mismatch")
+                size = sum(a["bytes"] for a in artifacts)
+                if size > self.config["task"]["submission"]["max_bytes"] or size < 1:
+                    raise RuntimeError("snapshot size outside package limit")
+            self.responses[phase] = response
+            self.write_journal()
+            return response
+        except BaseException as error:
+            self.failure = {"phase": phase, "error": type(error).__name__}
+            self.write_journal()
+            raise
+
+    def write_journal(self):
+        # Never persist tool tokens or opaque management handles.
+        phases = {p: {"request_id": r["request_id"], "status": r["status"]} for p, r in self.responses.items()}
+        self.journal.parent.mkdir(parents=True, exist_ok=True)
+        self.journal.write_text(json.dumps({"schema_version": "1", "phases": phases, "failure": self.failure}, indent=2))
+
+    async def prepare(self):
+        self.started = True
+        if self.config["task"]["driver"]["kind"] != "tool-server":
+            self.write_journal()
+            return
+        response = await self.phase("prepare")
+        if self.config["task"]["driver"]["config"].get("native_phases"):
+            # There is no candidate identity yet. Only the phase supervisor may
+            # bind and upload a phase token after prepare/setup have completed.
+            phases = self.config["task"]["driver"]["config"]["native_phases"]
+            if (response["output"].get("native_phases_ready") is not True or response["output"].get("tool_bindings", []) != []
+                    or phases["protocol"] == "hitch-native-phase-control@2" and response["output"].get("native_deadline_ready") is not True):
+                self.failure = {"phase": "prepare", "error": "NativePhasePrepareInvalid"}
+                self.write_journal()
+                raise RuntimeError("native phase prepare must confirm readiness without a static binding")
+            return
+        bindings = response["output"]["tool_bindings"]
+        expected = self.config["task"]["driver"]["config"]
+        if len(bindings) != 1 or bindings[0]["endpoint"] != expected["endpoint"] or bindings[0]["tools"] != self.config["tools"]:
+            raise RuntimeError("prepared tool binding differs from locked definition")
+        binding = bindings[0]
+        if not isinstance(binding.get("token"), str) or len(binding["token"]) < 32:
+            raise RuntimeError("missing per-trial tool authorization")
+        with tempfile.TemporaryDirectory(prefix="hitch-binding-") as temp:
+            file = Path(temp) / "binding.json"
+            file.write_text(json.dumps(binding)); file.chmod(0o600)
+            await self.env.upload_file(file, "/tmp/hitch-tool-binding.json")
+        await self.env.upload_file(Path(__file__).with_name("hitch_tool_client.mjs"), "/tmp/hitch-tools.mjs")
+
+    async def snapshot(self):
+        if self.stopped:
+            return
+        if self.config["task"]["driver"]["kind"] != "tool-server":
+            # Harbor collects the native task's declared artifacts. Shared
+            # verifiers retain upstream live-workspace semantics.
+            self.stopped = True
+            return
+        async def collect():
+            await self.phase("quiesce")
+            await self.phase("snapshot")
+        budget = self.config["profile"]["budget"]["collection_timeout_ms"]
+        await asyncio.wait_for(collect(), budget / 1000)
+        self.stopped = True
+
+    async def cleanup(self):
+        if self.started and self.config["task"]["driver"]["kind"] == "tool-server":
+            await self.phase("cleanup")
+
+
+def candidate_instruction(instruction, environment):
+    config = descriptor(environment.environment_dir)
+    if config is None:
+        return instruction, None
+    if config["task"]["driver"]["kind"] != "tool-server":
+        return (json.dumps(config["candidate_input"]) if "candidate_input" in config else instruction), int(config["agent_timeout_sec"] * 1000)
+    instruction += "\n\nTools are available through the locked tool-server bridge. Run `node /tmp/hitch-tools.mjs list` to read tool descriptions and JSON schemas. Invoke a tool using `node /tmp/hitch-tools.mjs TOOL_NAME 'JSON_ARGUMENTS'` (or pass - and JSON via stdin). Complete the requested workflow with these simulated service tools.\n"
+    if "tool-result-images@1" in config["task"]["requirements"]:
+        instruction += "Image tool results contain absolute local paths. Open each image with your native image viewing tool to inspect the observation. The files contain the original image bytes; the JSON metadata is not a visual observation.\n"
+    return instruction, int(config["agent_timeout_sec"] * 1000)
+
+
+async def export_final_response(environment, result):
+    config = descriptor(environment.environment_dir)
+    target = config["task"]["submission"].get("final_response") if config else None
+    if not target:
+        return
+    if not isinstance(result.get("output"), str):
+        raise RuntimeError("canonical candidate final response is unavailable")
+    with tempfile.TemporaryDirectory(prefix="hitch-response-") as temp:
+        source = Path(temp) / "response.json"
+        encoded = json.dumps({"schema_version": "1", "run_id": result["run_id"],
+            "response": result["output"], "termination": result["status"], "source": "hitch-run-result"})
+        if len(encoded.encode()) > config["task"]["submission"]["max_bytes"]:
+            raise RuntimeError("canonical response exceeds submission limit")
+        source.write_text(encoded)
+        # Keep an authoritative copy outside the candidate environment. A
+        # background process cannot replace the response used for grading.
+        (environment.trial_paths.trial_dir / "hitch-final-response.json").write_text(encoded)
+        prepared = await environment.exec("mkdir -p /hitch-evidence")
+        if prepared.return_code != 0:
+            raise RuntimeError("cannot prepare candidate response export")
+        await environment.upload_file(source, target)
+
+
+async def restore_final_response(verifier):
+    config = descriptor(verifier.task.paths.environment_dir)
+    target = config["task"]["submission"].get("final_response") if config else None
+    if not target:
+        return
+    source = verifier.trial_paths.trial_dir / "hitch-final-response.json"
+    if not source.is_file():
+        raise RuntimeError("trusted candidate response evidence is missing")
+    # Called after Harbor uploads the frozen artifacts, in the separate grader.
+    prepared = await verifier.environment.exec("rm -rf -- /hitch-evidence && mkdir -p /hitch-evidence")
+    if prepared.return_code != 0:
+        raise RuntimeError("cannot restore trusted candidate response")
+    await verifier.environment.upload_file(source, target)
+
+
+def validate_collected_submission(verifier):
+    config = descriptor(verifier.task.paths.environment_dir)
+    if not config or config["task"]["driver"]["kind"] == "tool-server":
+        return
+    task = config["task"]
+    if "separate-verifier" not in task["requirements"]:
+        return
+    # Harbor 0.21 mirrors absolute sources directly under artifacts/.
+    root = verifier.trial_paths.trial_dir / "artifacts"
+    seen, size = set(), 0
+    for source in task["submission"]["paths"]:
+        file = root / source.lstrip("/")
+        for ancestor in [file, *file.parents]:
+            if ancestor == root.parent:
+                break
+            if ancestor.is_symlink():
+                raise RuntimeError("collected submission contains a symlink")
+        if not file.exists():
+            raise RuntimeError(f"submission_missing: {source}")
+        for entry in [file, *(file.rglob("*") if file.is_dir() else [])]:
+            if entry.is_symlink():
+                raise RuntimeError("collected submission contains a symlink")
+            if entry.is_file() and entry not in seen:
+                seen.add(entry)
+                size += entry.stat().st_size
+                if size > task["submission"]["max_bytes"]:
+                    raise RuntimeError("collected submission exceeds package limit")
+
+
+def normalize_rewards(verifier, result):
+    config = descriptor(verifier.task.paths.environment_dir)
+    if config is None:
+        return result
+    journal = json.loads((verifier.trial_paths.trial_dir / "benchmark-lifecycle.json").read_text())
+    native = config["task"]["driver"]["kind"] != "tool-server"
+    if journal["failure"] or (not native and not {"prepare", "quiesce", "snapshot"} <= journal["phases"].keys()):
+        raise RuntimeError("benchmark lifecycle did not produce a valid snapshot")
+    directory = verifier.trial_paths.verifier_dir
+    upstream = config["task"]["grading"]["kind"] == "harbor"
+    if (directory / "reward.txt").exists() and not upstream:
+        raise RuntimeError("standard packages require only reward.json")
+    # Read the original JSON before Harbor/Pydantic numeric coercion: bools and
+    # numeric strings must not become valid binary scores via the result model.
+    if (directory / "reward.json").is_file():
+        raw = json.loads((directory / "reward.json").read_text())
+    elif upstream:
+        raw = {"reward": float((directory / "reward.txt").read_text().strip())}
+    else:
+        raise RuntimeError("metric_missing: reward.json")
+    if not isinstance(raw, dict):
+        raise RuntimeError("invalid grader metric object")
+    mapped = {}
+    for name, metric in config["metrics"].items():
+        field = config["task"]["grading"]["metric_map"][name]
+        if field not in raw:
+            raise RuntimeError(f"metric_missing: {field}")
+        value = raw[field]
+        if type(value) not in (float, int) or not math.isfinite(value) or not metric["range"][0] <= value <= metric["range"][1] or (metric["type"] == "binary" and value not in (0, 1)):
+            raise RuntimeError(f"metric_invalid: {field}")
+        mapped[name] = value
+    (directory / "benchmark-rewards.json").write_text(json.dumps({"raw": raw, "metrics": mapped, "primary_metric": config["primary_metric"], "source_task_id": config["task"]["source_task_id"], "task_digest": config["task_digest"]}, indent=2))
+    contract = config.get("score_contract", {"total_score": config["primary_metric"]})
+    if not isinstance(contract, dict) or set(contract) != {"total_score"} or contract["total_score"] not in mapped:
+        raise RuntimeError("invalid standardized score contract")
+    total = mapped[contract["total_score"]]
+    auxiliary = {name: value for name, value in raw.items() if name not in {"reward", "total_score", "process_score"}}
+    return result.model_copy(update={"rewards": {**auxiliary, "reward": total, "total_score": total}})

package/integrations/harbor/hitch_candidate_recycle.py

@@ -0,0 +1,262 @@
+"""Retire a candidate container without resetting the benchmark's sidecars.
+
+Trusted host API only. The caller must revoke the old tool binding and finish
+exporting the old run before calling this API, then reinstall the harness and
+bind the next run before starting a model. Receipts prove environment changes,
+not model conversation freshness, phase completion, or benchmark correctness.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+import os
+import re
+from pathlib import Path
+from typing import Any
+
+
+class CandidateRecycleError(RuntimeError):
+    pass
+
+
+def _overlaps(left: Path, right: Path) -> bool:
+    return left == right or left in right.parents or right in left.parents
+
+
+def _directory(path: Path) -> Path:
+    absolute = path.absolute()
+    if absolute.resolve(strict=True) != absolute or not absolute.is_dir():
+        raise CandidateRecycleError("Candidate phase paths must be real directories without symlinks")
+    return absolute
+
+
+def _write_json(path: Path, value: Any) -> None:
+    temporary = path.with_suffix(".pending")
+    with temporary.open("x", encoding="utf-8") as stream:
+        os.chmod(temporary, 0o600)
+        json.dump(value, stream, sort_keys=True, indent=2, allow_nan=False)
+        stream.write("\n")
+        stream.flush()
+        os.fsync(stream.fileno())
+    temporary.replace(path)
+
+
+async def _docker(arguments: list[str]) -> str:
+    process = await asyncio.create_subprocess_exec(
+        "docker", *arguments, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+    )
+    try:
+        stdout, _ = await asyncio.wait_for(process.communicate(), timeout=60)
+    except BaseException:
+        if process.returncode is None:
+            process.kill()
+        await process.wait()
+        raise
+    if process.returncode:
+        # Docker errors can contain environment values. Never journal them.
+        raise CandidateRecycleError("Candidate Docker inspection failed")
+    return stdout.decode("utf-8")
+
+
+_INSPECT = ('{"id":{{json .Id}},"image":{{json .Image}},'
+            '"state":{{json .State.Status}},"started_at":{{json .State.StartedAt}},'
+            '"labels":{{json .Config.Labels}},"mounts":{{json .Mounts}},'
+            '"host":{{json .HostConfig}},"config":{{json .Config}}}')
+
+
+def _configuration_digest(container: dict[str, Any]) -> str:
+    # Compare configuration without persisting potentially secret environment
+    # values. Compose regenerates its own config-hash on the image-pin overlay.
+    config = dict(container["config"])
+    config.pop("Image", None)
+    config.pop("Hostname", None)
+    environment = config.get("Env") or []
+    # Docker/Compose may reorder map-derived env and bind lists. Unique env
+    # keys have no ordering semantics; duplicate keys must retain their order.
+    if len({entry.split("=", 1)[0] for entry in environment}) == len(environment):
+        config["Env"] = sorted(environment)
+    config["Labels"] = {key: value for key, value in (config.get("Labels") or {}).items()
+                        if not key.startswith("com.docker.compose.") and key != "io.hitch.candidate-phase"}
+    host = dict(container["host"])
+    # These two are generated per container by Docker, not requested resources.
+    host.pop("ContainerIDFile", None)
+    if host.get("Binds"):
+        host["Binds"] = sorted(host["Binds"])
+    digest = hashlib.sha256(json.dumps([config, host], sort_keys=True).encode()).hexdigest()
+    return f"sha256:{digest}"
+
+
+class CandidateRecycler:
+    def __init__(self, environment: Any):
+        self.environment = environment
+        self.lock = asyncio.Lock()
+
+    async def _compose(self, arguments: list[str]) -> str:
+        result = await self.environment._run_docker_compose_command(
+            arguments, check=False, timeout_sec=60
+        )
+        if result.return_code:
+            raise CandidateRecycleError("Candidate Compose operation failed; trial cleanup required")
+        return result.stdout or ""
+
+    async def _containers(self) -> dict[str, dict[str, Any]]:
+        ids = (await self._compose(["ps", "--all", "--quiet"])).split()
+        if not ids or len(ids) > 64 or any(not re.fullmatch(r"[0-9a-f]{64}", item) for item in ids):
+            raise CandidateRecycleError("Candidate project container inventory is invalid")
+        documents = await _docker(["inspect", "--format", _INSPECT, *ids])
+        containers: dict[str, dict[str, Any]] = {}
+        ownership = self.environment._hitch_ownership_labels
+        if not ownership:
+            raise CandidateRecycleError("Candidate recycling requires Hitch lease ownership")
+        projects = set()
+        for line in documents.splitlines():
+            item = json.loads(line)
+            labels = item["labels"] or {}
+            if any(labels.get(key) != value for key, value in ownership.items()):
+                raise CandidateRecycleError("Candidate project lease ownership changed")
+            service = labels.get("com.docker.compose.service")
+            projects.add(labels.get("com.docker.compose.project"))
+            if not service or service in containers or labels.get("com.docker.compose.oneoff", "False").lower() != "false":
+                raise CandidateRecycleError("Candidate recycling requires one container per service")
+            containers[service] = item
+        if len(containers) != len(ids) or "main" not in containers or len(projects) != 1 or None in projects:
+            raise CandidateRecycleError("Candidate project inventory is incomplete")
+        return containers
+
+    @staticmethod
+    def _sidecars(containers: dict[str, dict[str, Any]]) -> dict[str, Any]:
+        result = {}
+        for service, item in containers.items():
+            if service != "main":
+                if item["state"] != "running":
+                    raise CandidateRecycleError("A benchmark sidecar is not running")
+                result[service] = {"id": item["id"], "image": item["image"], "started_at": item["started_at"]}
+        return result
+
+    def _mounts(self, containers: dict[str, dict[str, Any]], archive: Path) -> dict[str, Path]:
+        paths = self.environment.trial_paths
+        allowed = {"/logs/agent": paths.agent_dir, "/logs/verifier": paths.verifier_dir,
+                   "/logs/artifacts": paths.artifacts_dir}
+        main = containers["main"]
+        host = main["host"]
+        if (host.get("Privileged") or host.get("VolumesFrom") or host.get("Devices") or host.get("CapAdd")
+                or host.get("PidMode") or host.get("IpcMode") in ("host",) or str(host.get("IpcMode", "")).startswith("container:")
+                or host.get("NetworkMode") == "host"):
+            raise CandidateRecycleError("Candidate container isolation configuration is unsupported")
+        writable = {}
+        all_sources = []
+        for mount in main["mounts"]:
+            if mount["Type"] == "tmpfs":
+                continue  # Recreated with the container.
+            if mount["Type"] != "bind":
+                raise CandidateRecycleError("Persistent candidate volumes are unsupported for phase recycling")
+            source = Path(mount["Source"]).absolute()
+            if source.resolve(strict=True) != source:
+                raise CandidateRecycleError("Candidate mount source contains a symlink")
+            if not source.is_file() and not source.is_dir():
+                raise CandidateRecycleError("Special-file candidate mounts are unsupported")
+            if _overlaps(source, archive):
+                raise CandidateRecycleError("Candidate can access the phase archive")
+            all_sources.append(source)
+            if mount["RW"]:
+                target = mount["Destination"]
+                if target not in allowed or source != Path(allowed[target]).absolute():
+                    raise CandidateRecycleError("Only Harbor trial log directories may be writable candidate mounts")
+                writable[target] = _directory(source)
+        if ("/logs/agent" not in writable or len(set(writable.values())) != len(writable)
+                or len({source.name for source in writable.values()}) != len(writable)):
+            raise CandidateRecycleError("Candidate phase log mounts are missing or aliased")
+        for source in writable.values():
+            if sum(_overlaps(source, other) for other in all_sources) != 1:
+                raise CandidateRecycleError("Candidate mount paths overlap")
+            for service, item in containers.items():
+                if service != "main" and any(mount["Type"] == "bind" and _overlaps(source, Path(mount["Source"])) for mount in item["mounts"]):
+                    raise CandidateRecycleError("Candidate logs are shared with a persistent service")
+        return writable
+
+    async def recycle(self, phase_index: int) -> dict[str, Any]:
+        if type(phase_index) is not int or not 1 <= phase_index <= 9999:
+            raise CandidateRecycleError("Candidate phase index must be an integer from 1 to 9999")
+        async with self.lock:
+            if getattr(self.environment, "_is_windows_container", False):
+                raise CandidateRecycleError("Candidate recycling supports Linux containers only")
+            trial = _directory(self.environment.trial_paths.trial_dir)
+            root = trial / "hitch-candidate-phases"
+            before = await self._containers()
+            mounts = self._mounts(before, root)
+            sidecars = self._sidecars(before)
+            phase = root / f"phase-{phase_index:04d}"
+            receipt_path = phase / "receipt.json"
+            root.mkdir(mode=0o700, exist_ok=True)
+            _directory(root)
+            if root.stat().st_mode & 0o077:
+                raise CandidateRecycleError("Candidate phase archive must be private to the host controller")
+            if receipt_path.exists():
+                _directory(phase)
+                if receipt_path.is_symlink():
+                    raise CandidateRecycleError("Candidate phase receipt is a symlink")
+                receipt = json.loads(receipt_path.read_text())
+                if (receipt.get("status") != "completed" or receipt.get("new_container_id") != before["main"]["id"]
+                        or receipt.get("sidecars") != sidecars or receipt.get("ownership") != self.environment._hitch_ownership_labels):
+                    raise CandidateRecycleError("Candidate recycle receipt is stale or incomplete; cleanup required")
+                return receipt
+            if phase.exists():
+                raise CandidateRecycleError("Candidate phase archive already exists; cleanup required")
+            existing = sorted(item.name for item in root.iterdir())
+            if existing != [f"phase-{index:04d}" for index in range(1, phase_index)]:
+                raise CandidateRecycleError("Candidate phases must be recycled consecutively")
+            if phase_index > 1:
+                previous = json.loads((root / f"phase-{phase_index - 1:04d}" / "receipt.json").read_text())
+                if (previous.get("status") != "completed" or previous.get("new_container_id") != before["main"]["id"]
+                        or previous.get("sidecars") != sidecars or previous.get("image") != before["main"]["image"]):
+                    raise CandidateRecycleError("Previous candidate replacement is not complete")
+            main = before["main"]
+            if not re.fullmatch(r"sha256:[0-9a-f]{64}", main["image"]):
+                raise CandidateRecycleError("Candidate image identity is invalid")
+            phase.mkdir(mode=0o700)
+            receipt = {"schema_version": "hitch-candidate-recycle@1", "scope": "environment-only",
+                       "phase_index": phase_index, "status": "prepared", "old_container_id": main["id"],
+                       "image": main["image"], "configuration_digest": _configuration_digest(main),
+                       "ownership": dict(self.environment._hitch_ownership_labels), "sidecars": sidecars,
+                       "archives": {target: f"phase-{phase_index:04d}/{source.name}" for target, source in mounts.items()}}
+            _write_json(receipt_path, receipt)
+            try:
+                # Deliberately bypass stop_service(): that hook snapshots the
+                # entire benchmark and would terminate the native phase loop.
+                await self._compose(["rm", "--stop", "--force", "--volumes", "main"])
+                remaining = set((await _docker(["ps", "--all", "--no-trunc", "--quiet"])).split())
+                if main["id"] in remaining:
+                    raise CandidateRecycleError("Previous candidate container still exists")
+                receipt["status"] = "retired"
+                _write_json(receipt_path, receipt)
+                for source in mounts.values():
+                    _directory(source)
+                    mode = source.stat().st_mode & 0o777
+                    source.rename(phase / source.name)
+                    source.mkdir(mode=mode)
+                    source.chmod(mode)
+                receipt["status"] = "archived"
+                _write_json(receipt_path, receipt)
+                overlay = phase / "replacement.json"
+                _write_json(overlay, {"services": {"main": {"image": main["image"], "labels": {"io.hitch.candidate-phase": str(phase_index + 1)}}}})
+                self.environment._hitch_phase_compose_path = overlay
+                await self._compose(["up", "--detach", "--wait", "--wait-timeout", "45", "--no-deps", "--no-build", "--pull", "never", "main"])
+                after = await self._containers()
+                next_main = after["main"]
+                checks = {"new-container": next_main["id"] != main["id"], "image": next_main["image"] == main["image"],
+                          "running": next_main["state"] == "running", "configuration": _configuration_digest(next_main) == receipt["configuration_digest"],
+                          "sidecars": self._sidecars(after) == sidecars, "mounts": self._mounts(after, root) == mounts}
+                failed_checks = [name for name, passed in checks.items() if not passed]
+                if failed_checks:
+                    raise CandidateRecycleError("Replacement changed the benchmark environment contract: " + ", ".join(failed_checks))
+                await self.environment.ensure_dirs(list(mounts))
+                await self.environment._upload_environment_dir_after_start()
+                receipt.update(status="completed", new_container_id=next_main["id"])
+                _write_json(receipt_path, receipt)
+                return receipt
+            except BaseException as error:
+                receipt.update(status="failed", failure_type=type(error).__name__)
+                _write_json(receipt_path, receipt)
+                raise