agent-hitch 0.2.7 → 0.2.9

package/integrations/harbor/hitch_harbor_agent.py

@@ -2,9 +2,12 @@
 
 from __future__ import annotations
 
+import asyncio
 import hashlib
 import json
+import os
 import re
+import secrets
 import shlex
 import stat as stat_module
 import tempfile
@@ -13,7 +16,7 @@ import uuid
 from urllib.parse import urlparse
 from datetime import datetime, timezone
 from pathlib import Path, PurePosixPath
-from typing import Any
+from typing import Any, NamedTuple
 
 from harbor.agents.base import BaseAgent
 from harbor.environments.base import BaseEnvironment, ExecResult
@@ -22,10 +25,69 @@ from harbor.models.agent.context import AgentContext
 CONTROLLER_RUNTIME_MANIFEST_VERSION = "2"
 HARNESS_ARTIFACT_REMOTE_ROOT = "/opt/hitch-harness-artifact"
 HITCH_BRIDGE_ERROR_LOG = "/logs/agent/hitch-bridge-error.json"
+HITCH_AGENT_OUTCOME_NAME = "hitch-agent-outcome.json"
 HITCH_DIAGNOSTIC_MAX_BYTES = 8 * 1024
 HITCH_BRIDGE_ERROR_MAX_BYTES = 64 * 1024
 HITCH_RESULT_MISSING_EXIT = 44
 HITCH_RESULT_NOT_FILE_EXIT = 45
+PHASE_EXPORT_MODULE = "dist/src/runs/phase-bundle.js"
+PHASE_CONTROL_MODULE = "dist/src/runs/phase-cancellation.js"
+
+
+def artifact_directory_integrity(directory: Path, captured_files: dict[str, bytes] | None = None) -> str:
+    """Match artifacts/integrity.ts, including modes and internal symlinks."""
+    root = Path(os.path.abspath(directory))
+    digest = hashlib.sha256()
+
+    def visit(parent: Path) -> None:
+        # JS Array.sort compares UTF-16 code units, not Unicode code points.
+        for entry in sorted(parent.iterdir(), key=lambda item: item.name.encode("utf-16-be", "surrogatepass")):
+            if parent == root and entry.name == "artifact.json":
+                continue
+            relative = entry.relative_to(root).as_posix()
+            info = entry.lstat()
+            mode = info.st_mode & 0o7777
+            if stat_module.S_ISDIR(info.st_mode):
+                digest.update(f"d\0{relative}\0{mode}\0".encode())
+                visit(entry)
+            elif stat_module.S_ISREG(info.st_mode):
+                digest.update(f"f\0{relative}\0{mode}\0{info.st_size}\0".encode())
+                capture = captured_files is not None and relative in captured_files
+                if capture and info.st_size > 16_384:
+                    raise RuntimeError("captured artifact metadata exceeds its size limit")
+                content = bytearray()
+                with entry.open("rb") as handle:
+                    for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+                        digest.update(chunk)
+                        if capture:
+                            content.extend(chunk)
+                            if len(content) > 16_384:
+                                raise RuntimeError("captured artifact metadata exceeds its size limit")
+                if capture and captured_files is not None:
+                    captured_files[relative] = bytes(content)
+                digest.update(b"\0")
+            elif stat_module.S_ISLNK(info.st_mode):
+                target = os.readlink(entry)
+                if not Path(os.path.abspath(entry.parent / target)).is_relative_to(root):
+                    raise RuntimeError("artifact symlink escapes the artifact directory")
+                digest.update(f"l\0{relative}\0{target}\0".encode())
+            else:
+                raise RuntimeError("artifact contains a special file")
+
+    visit(root)
+    return "sha256:" + digest.hexdigest()
+
+
+class PreparedPhase(NamedTuple):
+    run_id: str
+    instruction: str
+    context_json: str
+    parent_json: str
+    identity: str
+    deadline_ns: int
+
+    def __repr__(self) -> str:
+        return f"PreparedPhase(run_id={self.run_id!r})"
 
 
 class HitchBridgeError(RuntimeError):
@@ -78,6 +140,7 @@ class HitchHarborAgent(BaseAgent):
         if not isinstance(node_version, str) or re.fullmatch(r"v\d+\.\d+\.\d+", node_version) is None:
             raise ValueError("node_version must be an exact stable Node.js version")
         self.required_node_version = node_version
+        self._node_bin_directory: str | None = None
         if local_source_transport is not None:
             raise ValueError("trial-side local source transports are no longer supported")
         self.candidate_id = candidate_id
@@ -108,6 +171,17 @@ class HitchHarborAgent(BaseAgent):
         self._artifact_host_directory: Path | None = None
         self._artifact_uploaded = False
         self._artifact_transport_status: str | None = None
+        self._setup_complete = False
+        self._phase_export_available = False
+        self._phase_supervision_available = False
+        self._prepared_phase: PreparedPhase | None = None
+        self._prepared_phase_keys: set[tuple[str, int]] = set()
+        self._phase_group_contracts: dict[str, tuple[str, str, int]] = {}
+        self._phase_inflight = False
+        self._active_phase: PreparedPhase | None = None
+        self._phase_cancel_lock: asyncio.Lock | None = None
+        self._phase_cancel_receipts: dict[str, dict[str, Any]] = {}
+        self._phase_control_tokens: dict[str, str] = {}
 
     @staticmethod
     def name() -> str:
@@ -118,8 +192,10 @@ class HitchHarborAgent(BaseAgent):
 
     async def setup(self, environment: BaseEnvironment) -> None:
         started_ns = time.monotonic_ns()
+        self._setup_complete = False
         try:
             await self._setup(environment)
+            self._setup_complete = True
         finally:
             self._write_phase_timing("setup", started_ns)
 
@@ -147,6 +223,8 @@ class HitchHarborAgent(BaseAgent):
         # and the actual container upload, spec §4.6).
         self._verify_manifest_identity(manifest)
         self._verify_payload(manifest)
+        self._phase_export_available = {PHASE_EXPORT_MODULE, PHASE_CONTROL_MODULE}.issubset({file.get("path") for file in manifest["files"]})
+        self._phase_supervision_available = self._phase_export_available and "integrations/harbor/hitch_phase_supervisor.py" in {file.get("path") for file in manifest["files"]}
         if self.harness_artifact is None:
             raise RuntimeError("hitch-artifact-materialize: Harbor requires a dedicated-builder artifact")
         try:
@@ -163,8 +241,7 @@ class HitchHarborAgent(BaseAgent):
         # bookkeeping and is not identity (spec §4.2).
         await environment.upload_dir(payload_dir, "/opt/hitch")
         await self._ensure_node(environment)
-        platform = await self._container_platform(environment)
-        node_version = await self._container_node_version(environment)
+        platform, node_version = await self._container_node_identity(environment)
         if self._artifact_manifest is None or not self._artifact_compatible(
             self._artifact_manifest, platform, node_version
         ):
@@ -369,22 +446,43 @@ class HitchHarborAgent(BaseAgent):
             raise RuntimeError(f"{label} exceeds the size limit ({maximum} bytes)")
         return info
 
-    async def _container_platform(self, environment: BaseEnvironment) -> str:
-        result = await self._exec(
-            environment,
-            f'{self._node_prefix()} node -p "process.platform + \'-\' + process.arch"',
+    async def _container_node_identity(self, environment: BaseEnvironment) -> tuple[str, str]:
+        # Harbor may merge stderr into stdout. Frame only the machine value so
+        # startup warnings cannot be mistaken for the platform or version.
+        marker = "__HITCH_NODE_IDENTITY__"
+        script = (
+            f"process.stdout.write('\\n{marker}' + process.platform + '-' + "
+            "process.arch + ' ' + process.version + '\\n')"
+        )
+        command = f"{self._node_prefix()} node -e {shlex.quote(script)}"
+        result = await environment.exec(command)
+        lines = [line for line in (result.stdout or "").splitlines() if line.startswith(marker)]
+        identity = re.fullmatch(
+            re.escape(marker) + r"((?:linux|darwin|win32)-[a-z0-9_]+) (v\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?)",
+            lines[0],
+        ) if len(lines) == 1 else None
+        if result.return_code == 0 and identity is not None:
+            return identity.group(1), identity.group(2)
+
+        reason = f"exit={result.return_code}" if result.return_code != 0 else "expected one valid identity marker"
+        failure = self._node_runtime_error(
+            "hitch_node_runtime_identity_invalid",
+            f"container returned an invalid Node.js identity ({reason}); "
+            f"stdout={self._bounded_tail(result.stdout or '', 512)!r}; "
+            f"stderr={self._bounded_tail(result.stderr or '', 512)!r}",
         )
-        platform = (result.stdout or "").strip()
-        if not re.fullmatch(r"(?:linux|darwin|win32)-[a-z0-9_]+", platform):
-            raise RuntimeError("container returned an invalid Node.js platform identity")
-        return platform
-
-    async def _container_node_version(self, environment: BaseEnvironment) -> str:
-        result = await self._exec(environment, f'{self._node_prefix()} node -p "process.version"')
-        version = (result.stdout or "").strip()
-        if not re.fullmatch(r"v\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?", version):
-            raise RuntimeError("container returned an invalid Node.js version")
-        return version
+        failure.evidence["probe"] = {
+            "command": command,
+            "return_code": result.return_code,
+            "stdout_tail": self._bounded_tail(result.stdout or ""),
+            "stderr_tail": self._bounded_tail(result.stderr or ""),
+        }
+        self._record_node_runtime({"source": "failed", "bin_directory": self._node_bin_directory, **failure.evidence})
+        try:
+            await self._write_bridge_error(environment, failure.evidence)
+        except Exception:
+            pass  # Keep the probe failure if the container cannot export logs.
+        raise failure
 
     @staticmethod
     def _artifact_compatible(manifest: dict[str, Any], platform: str, node_version: str) -> bool:
@@ -513,26 +611,209 @@ class HitchHarborAgent(BaseAgent):
         environment: BaseEnvironment,
         context: AgentContext,
     ) -> None:
+        session = getattr(environment, "_hitch_benchmark", None)
+        config = session.config if session else None
+        driver = config["task"]["driver"] if config else None
+        phases = driver["config"].get("native_phases") if driver and driver["kind"] == "tool-server" else None
+        if phases:
+            from hitch_phase_supervisor import NativePhaseSupervisor
+            task_timeout = int(config["agent_timeout_sec"] * 1000)
+            remaining = min(task_timeout, self.hitch_timeout_ms) if self.hitch_timeout_ms > 0 else task_timeout
+            result = await NativePhaseSupervisor(
+                self, environment, controller={"service": driver["config"]["service"], "argv": phases["argv"]},
+                binding={"endpoint": driver["config"]["endpoint"], "tools": config["tools"]},
+                task_digest=config["task_digest"], timeout_ms=remaining,
+                shutdown_timeout_ms=phases["shutdown_timeout_ms"],
+                finalization_timeout_ms=phases.get("finalization_timeout_ms"),
+                task_instruction=instruction,
+            ).run()
+            context.metadata = {"candidate_id": self.candidate_id, "harness_ref": self.harness_ref,
+                                "revision_identity": self.revision_identity, "controller_runtime_id": self.controller_runtime_id,
+                                "hitch_context_kind": "benchmark_phase_group", "hitch_run_group_id": result["run_group_id"],
+                                "hitch_phase_count": len(result["phases"]), "hitch_status": "succeeded",
+                                "hitch_phase_supervision": "hitch-native-phases/supervision.json"}
+            return
         started_ns = time.monotonic_ns()
         try:
             await self._run(instruction, environment, context)
         finally:
             self._write_phase_timing("agent", started_ns)
 
+    def _phase_identity(self) -> str:
+        identity = [self.candidate_id, self.harness_ref, self.revision_identity, self.controller_runtime_id,
+                    self.model_name, self.agent_args, self.credential_names, self._require_workdir(),
+                    self.eval_id, self.benchmark_id, self.benchmark_revision, self.verifier_identity, self._trial_identity()]
+        return hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest()
+
+    def prepare_phase(self, *, instruction: str, run_group_id: str, phase_index: int,
+                      task_digest: str, remaining_timeout_ms: int) -> PreparedPhase:
+        """Reserve identity before private tool binding; does not start a model.
+
+        The supervisor supplies one frozen task digest across all phases and a
+        remaining whole-task budget. Time spent binding/uploading consumes that
+        budget too. The returned immutable handle is single-use on this agent.
+        """
+        if not self._setup_complete or not self._phase_export_available:
+            raise RuntimeError("phase preparation requires setup with a phase-export capable runtime")
+        if self._prepared_phase is not None or self._phase_inflight:
+            raise RuntimeError("another candidate phase is already prepared or running")
+        if (not isinstance(instruction, str) or not instruction.strip()
+                or not isinstance(run_group_id, str) or not re.fullmatch(r"run_group_[a-f0-9]{32}", run_group_id)
+                or type(phase_index) is not int or not 1 <= phase_index <= 10000
+                or not isinstance(task_digest, str) or not re.fullmatch(r"sha256:[a-f0-9]{64}", task_digest)
+                or type(remaining_timeout_ms) is not int or not 1 <= remaining_timeout_ms <= 9007199254740991):
+            raise ValueError("invalid candidate phase identity or remaining budget")
+        if not all((self.eval_id, self.benchmark_id, self.benchmark_revision, self.verifier_identity)):
+            raise ValueError("candidate phases require a complete eval and benchmark identity")
+        if (not re.fullmatch(r"eval_[a-f0-9]{32}", self.eval_id)
+                or any(not re.fullmatch(r"sha256:[a-f0-9]{64}", value) for value in (self.benchmark_revision, self.verifier_identity))):
+            raise ValueError("candidate phase eval and benchmark identities must be immutable")
+        key = (run_group_id, phase_index)
+        if key in self._prepared_phase_keys:
+            raise RuntimeError("candidate phase identity was already prepared; implicit retries are forbidden")
+        identity = self._phase_identity()
+        previous = self._phase_group_contracts.get(run_group_id)
+        if (previous is None and phase_index != 1
+                or previous is not None and previous != (task_digest, identity, phase_index - 1)):
+            raise RuntimeError("candidate phase group must retain its task/candidate identity and consecutive indices")
+        trial_id, task_id, attempt = self._trial_identity()
+        context = {"kind": "benchmark_phase", "benchmark_id": self.benchmark_id,
+                   "benchmark_revision": self.benchmark_revision, "task_id": task_id,
+                   "task_digest": task_digest, "verifier_identity": self.verifier_identity,
+                   "run_group_id": run_group_id, "phase_index": phase_index}
+        parent = {"kind": "eval", "eval_id": self.eval_id, "trial_id": trial_id, "attempt": attempt}
+        prepared = PreparedPhase("run_" + uuid.uuid4().hex, instruction, json.dumps(context, sort_keys=True),
+                                 json.dumps(parent, sort_keys=True), identity,
+                                 time.monotonic_ns() + remaining_timeout_ms * 1_000_000)
+        self._phase_control_tokens[prepared.run_id] = secrets.token_hex(32)
+        self._prepared_phase_keys.add(key)
+        self._phase_group_contracts[run_group_id] = (task_digest, identity, phase_index)
+        self._prepared_phase = prepared
+        return prepared
+
+    async def run_phase(self, prepared: PreparedPhase, environment: BaseEnvironment, context: AgentContext) -> None:
+        if not isinstance(prepared, PreparedPhase) or self._prepared_phase is not prepared or self._phase_inflight:
+            raise RuntimeError("candidate phase handle is stale, foreign, or already consumed")
+        self._prepared_phase = None  # Every outcome consumes the handle.
+        if prepared.identity != self._phase_identity():
+            self._phase_control_tokens.pop(prepared.run_id, None)
+            raise RuntimeError("candidate identity changed after phase preparation")
+        if prepared.deadline_ns <= time.monotonic_ns():
+            self._phase_control_tokens.pop(prepared.run_id, None)
+            raise RuntimeError("candidate whole-task budget expired before phase start")
+        self._phase_inflight = True
+        self._active_phase = prepared
+        started_ns = time.monotonic_ns()
+        try:
+            await self._run(prepared.instruction, environment, context, prepared_phase=prepared)
+        finally:
+            self._phase_inflight = False
+            self._active_phase = None
+            self._phase_control_tokens.pop(prepared.run_id, None)
+            self._write_phase_timing("agent", started_ns)
+
+    @staticmethod
+    def _phase_control_path(prepared: PreparedPhase) -> str:
+        return f"/tmp/hitch-phase-control-{prepared.run_id}.config.json"
+
+    @staticmethod
+    async def _upload_phase_json(environment: BaseEnvironment, target: str, value: dict[str, Any]) -> None:
+        # Keep control nonces out of both argv and the mounted agent log tree.
+        with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", prefix="hitch-phase-control-", delete=False) as handle:
+            json.dump(value, handle)
+            temporary = Path(handle.name)
+        try:
+            await environment.upload_file(temporary, target)
+            result = await environment.exec(f"chmod 600 {shlex.quote(target)}")
+            if result.return_code != 0:
+                raise RuntimeError("could not protect candidate phase control input")
+        finally:
+            temporary.unlink(missing_ok=True)
+
+    async def request_phase_cancellation(self, prepared: PreparedPhase, environment: BaseEnvironment, *, reason: str) -> dict[str, Any]:
+        """Request executor cancellation; await run_phase separately for sealed evidence.
+
+        This receipt proves only that the request was delivered, not that the
+        model stopped or that a native phase completed. Await this operation
+        before recycling the candidate. A caller must separately enforce a
+        bounded shutdown/collection allowance and whole-trial failure cleanup.
+        """
+        if reason not in {"native_phase_reset", "native_task_finished", "task_budget_expired", "cancelled"}:
+            raise ValueError("invalid phase cancellation reason")
+        if self._phase_cancel_lock is None:
+            self._phase_cancel_lock = asyncio.Lock()
+        async with self._phase_cancel_lock:
+            if not isinstance(prepared, PreparedPhase) or self._active_phase is not prepared:
+                raise RuntimeError("candidate phase is not active")
+            existing = self._phase_cancel_receipts.get(prepared.run_id)
+            if existing is not None:
+                if existing["reason"] != reason:
+                    raise RuntimeError("candidate phase cancellation reason already fixed")
+                if existing["status"] != "delivered":
+                    raise RuntimeError("candidate cancellation delivery is incomplete; implicit retries are forbidden")
+                return dict(existing)
+            phase = json.loads(prepared.context_json)
+            record_dir = self.logs_dir.parent / "hitch-phase-control"
+            record_dir.mkdir(mode=0o700, exist_ok=True)
+            if record_dir.is_symlink() or record_dir.stat().st_mode & 0o077:
+                raise RuntimeError("candidate cancellation records require a private host directory")
+            record_path = record_dir / f"{prepared.run_id}.request.json"
+            receipt = {"schema_version": "hitch-phase-cancel-request@1", "scope": "request-only",
+                       "status": "prepared",
+                       "request_id": "phase_cancel_" + uuid.uuid4().hex, "run_id": prepared.run_id,
+                       "run_group_id": phase["run_group_id"], "phase_index": phase["phase_index"],
+                       "reason": reason, "requested_at": datetime.now(timezone.utc).isoformat(),
+                       "record_ref": f"hitch-phase-control/{record_path.name}"}
+            with record_path.open("x", encoding="utf-8") as handle:
+                json.dump(receipt, handle, sort_keys=True)
+                handle.write("\n")
+            record_path.chmod(0o600)
+            self._phase_cancel_receipts[prepared.run_id] = receipt
+            try:
+                await self._upload_phase_json(environment, self._phase_control_path(prepared).replace(".config.json", ".request.json"), {
+                    "schema_version": "hitch-phase-cancel@1", "run_id": prepared.run_id,
+                    "token": self._phase_control_tokens[prepared.run_id], "reason": reason,
+                })
+                receipt["status"] = "delivered"
+            except BaseException as error:
+                receipt.update(status="delivery_failed", failure_type=type(error).__name__)
+                raise
+            finally:
+                update = record_path.with_suffix(".pending")
+                with update.open("x", encoding="utf-8") as handle:
+                    json.dump(receipt, handle, sort_keys=True)
+                    handle.write("\n")
+                update.chmod(0o600)
+                update.replace(record_path)
+            return dict(receipt)
+
     async def _run(
         self,
         instruction: str,
         environment: BaseEnvironment,
         context: AgentContext,
+        *,
+        prepared_phase: PreparedPhase | None = None,
     ) -> None:
+        invocation_started_ns = time.monotonic_ns()
+        session = getattr(environment, "_hitch_benchmark", None)
+        task_budget_ms = self.hitch_timeout_ms
+        if prepared_phase is None and session:
+            from hitch_benchmark import candidate_instruction
+            instruction, task_timeout = candidate_instruction(instruction, environment)
+            if task_timeout is not None:
+                task_budget_ms = min(task_timeout, task_budget_ms) if task_budget_ms > 0 else task_timeout
         self.logs_dir.mkdir(parents=True, exist_ok=True)
         workdir = self._require_workdir()
-        assigned_run_id = "run_" + uuid.uuid4().hex
+        assigned_run_id = prepared_phase.run_id if prepared_phase else "run_" + uuid.uuid4().hex
         run_id = assigned_run_id
         trial_id, task_id, attempt = self._trial_identity()
         context_payload: dict[str, Any] = {"kind": "ad_hoc"}
         parent_payload: dict[str, Any] | None = None
-        if all((self.eval_id, self.benchmark_id, self.benchmark_revision, self.verifier_identity)):
+        if prepared_phase is not None:
+            context_payload = json.loads(prepared_phase.context_json)
+            parent_payload = json.loads(prepared_phase.parent_json)
+        elif all((self.eval_id, self.benchmark_id, self.benchmark_revision, self.verifier_identity)):
             workspace_digest = await self._workspace_digest(environment)
             task_digest_input = json.dumps(
                 {
@@ -590,14 +871,35 @@ class HitchHarborAgent(BaseAgent):
                 parent_temporary.unlink(missing_ok=True)
 
         proxy_environment, proxy_health = await self._model_proxy_environment(environment, run_id)
+        if prepared_phase is not None:
+            await self._upload_phase_json(environment, self._phase_control_path(prepared_phase), {
+                "schema_version": "hitch-phase-control@1", "run_id": run_id, "token": self._phase_control_tokens[run_id],
+            })
         if self._entrypoint is None:
             raise RuntimeError("Hitch agent setup() must run before run() to resolve the runtime entrypoint")
         entry = self._remote_entry(self._entrypoint)
+        timeout_ms = task_budget_ms
+        if prepared_phase is not None:
+            timeout_ms = (prepared_phase.deadline_ns - time.monotonic_ns()) // 1_000_000
+            if timeout_ms <= 0:
+                raise RuntimeError("candidate whole-task budget expired during phase binding/upload")
+        elif session:
+            preparation_ms = (time.monotonic_ns() - invocation_started_ns) // 1_000_000
+            timeout_ms = task_budget_ms - preparation_ms
+            (self.logs_dir / "hitch-agent-budget.json").write_text(json.dumps({
+                "schema_version": "hitch-agent-budget@1", "run_id": run_id,
+                "task_budget_ms": task_budget_ms, "preparation_ms": preparation_ms,
+                "hitch_timeout_ms": max(0, timeout_ms),
+                "collection_timeout_ms": session.config["profile"]["budget"]["collection_timeout_ms"],
+                "scope": "invocation-budget-and-collection-allowance",
+            }))
+            if timeout_ms <= 0:
+                raise RuntimeError("candidate budget expired during input preparation; no model was launched")
         arguments = [
             self._node_prefix(),
             "HITCH_ROOT=/tmp/hitch-state",
             *proxy_environment,
-            *(["HITCH_HARBOR_INTERNAL=1"] if self._artifact_uploaded or parent_payload is not None else []),
+            "HITCH_HARBOR_INTERNAL=1",
             f"node {entry} run",
             "--harness",
             shlex.quote(self.harness_ref),
@@ -611,29 +913,138 @@ class HitchHarborAgent(BaseAgent):
             "--context-file",
             "/tmp/hitch-context.json",
             "--timeout",
-            str(self.hitch_timeout_ms),
+            str(timeout_ms),
             "--output",
             "jsonl",
+            "--internal-run-id", run_id,
         ]
         if parent_payload is not None:
             arguments.extend([
                 "--parent-file", "/tmp/hitch-parent.json",
-                "--internal-run-id", run_id,
-                "--internal-defer-benchmark-observation",
             ])
+            if prepared_phase is None:
+                arguments.append("--internal-defer-benchmark-observation")
+            else:
+                arguments.extend(["--internal-phase-control", shlex.quote(self._phase_control_path(prepared_phase))])
         if self.model_name:
             arguments.extend(["--model", shlex.quote(self.model_name)])
         for value in self.agent_args:
             arguments.extend(["--agent-arg", shlex.quote(value)])
         for name in self.credential_names:
             arguments.extend(["--internal-credential-name", shlex.quote(name)])
-        command = (
-            "set -o pipefail; "
-            + " ".join(arguments)
-            + " 2> >(tee /logs/agent/hitch-stderr.log >&2)"
-            + " | tee /logs/agent/hitch-events.jsonl"
+        command = self._logged_run_command(" ".join(arguments))
+        try:
+            execution = await environment.exec(command, cwd=workdir)
+        except (Exception, asyncio.CancelledError):
+            # Harbor cancels this await on its agent deadline. Collect what is
+            # already durable before it removes the container, without turning
+            # a cancelled execution into a successful observation.
+            await self._preserve_interrupted_run(environment, assigned_run_id)
+            raise
+        collection = self._collect_run(
+            environment, context, execution, assigned_run_id=assigned_run_id,
+            context_payload=context_payload, parent_payload=parent_payload, prepared_phase=prepared_phase,
+            workdir=workdir, proxy_health=proxy_health,
         )
-        execution = await environment.exec(command, cwd=workdir)
+        if prepared_phase is not None or not session:
+            await collection
+            return
+        try:
+            await asyncio.wait_for(collection, session.config["profile"]["budget"]["collection_timeout_ms"] / 1000)
+        except asyncio.TimeoutError as error:
+            # A completed process with uncollected evidence is still invalid.
+            # Do not fabricate a terminal bundle or relabel it as model success.
+            receipt = {"code": "hitch_run_collection_timeout", "run_id": assigned_run_id,
+                       "process_return_code": execution.return_code}
+            (self.logs_dir / "hitch-collection-timeout.json").write_text(json.dumps(receipt))
+            raise RuntimeError("hitch_run_collection_timeout: terminal evidence export exceeded its allowance") from error
+
+    @staticmethod
+    def _logged_run_command(invocation: str) -> str:
+        # Bash process substitution leaves an extra pipe FD in the harness and
+        # its descendants. A task's background service may outlive the harness.
+        # Regular files keep live logs without tying exec completion to that FD.
+        return (
+            "hitch_run_exit=0; " + invocation
+            + " > /logs/agent/hitch-events.jsonl 2> /logs/agent/hitch-stderr.log || hitch_run_exit=$?; "
+            + "cat /logs/agent/hitch-events.jsonl || exit $?; "
+            + "cat /logs/agent/hitch-stderr.log >&2 || exit $?; "
+            + 'exit "$hitch_run_exit"'
+        )
+
+    async def _preserve_interrupted_run(self, environment: BaseEnvironment, run_id: str) -> None:
+        # This is a diagnostic directory, never a complete/sealed run bundle.
+        # runtime-home contains credentials and must not be copied. Only files
+        # already produced by Hitch's redacted evidence writers are eligible.
+        destination = f"/logs/agent/hitch-interrupted-run-{uuid.uuid4().hex}"
+        script = r"""
+import {lstat,mkdir,open,readFile,writeFile} from 'node:fs/promises';
+import {constants} from 'node:fs';
+import path from 'node:path';
+const [source,target]=process.argv.slice(1);
+let remaining=64*1024*1024;
+const copied=[];
+await mkdir(target,{mode:0o700});
+async function copy(relative){
+  if(copied.includes(relative))return;
+  if(typeof relative!=='string'||relative.includes('\\')||path.isAbsolute(relative)
+    ||relative.split('/').some(p=>!p||p==='.'||p==='..')) return;
+  const file=path.join(source,relative);
+  try {
+    for(let p=path.dirname(file);p!=='/';p=path.dirname(p)) {
+      const info=await lstat(p); if(!info.isDirectory()||info.isSymbolicLink()) return;
+    }
+    if(!(await lstat(file)).isFile())return;
+    const input=await open(file,constants.O_RDONLY|constants.O_NOFOLLOW|constants.O_NONBLOCK);
+    try {
+      const info=await input.stat();
+      if(!info.isFile()||info.nlink!==1||info.size>remaining) return;
+      const bytes=Buffer.alloc(info.size);
+      let offset=0;
+      while(offset<bytes.length){const r=await input.read(bytes,offset,bytes.length-offset,offset);if(!r.bytesRead)break;offset+=r.bytesRead;}
+      remaining-=offset;
+      const output=path.join(target,relative);
+      await mkdir(path.dirname(output),{recursive:true,mode:0o700});
+      await writeFile(output,bytes.subarray(0,offset),{flag:'wx',mode:0o600});
+      copied.push(relative);
+    } finally {await input.close();}
+  } catch(e) {if(!['ENOENT','ELOOP','ENOTDIR'].includes(e.code))throw e;}
+}
+for(const name of ['result.json','manifest.json','events.jsonl','stdout.log','stderr.log','trajectory.ref.json']) await copy(name);
+if(copied.includes('trajectory.ref.json')) {
+  try {
+    const ref=JSON.parse(await readFile(path.join(target,'trajectory.ref.json'),'utf8'));
+    const files=Array.isArray(ref.files)?ref.files.map(f=>f?.path):[ref.path];
+    for(const file of files.slice(0,256)) if(typeof file==='string'&&file.startsWith('trajectory/'))await copy(file);
+  } catch(e) {if(!(e instanceof SyntaxError))throw e;}
+}
+// ProviderCaptureWriter persists redacted output before finalization creates the reference.
+for(const name of ['trajectory/provider/events.jsonl','trajectory/provider/transcript.txt']) await copy(name);
+await writeFile(path.join(target,'diagnostic.json'),JSON.stringify({complete:false,copied}),{flag:'wx',mode:0o600});
+""".strip()
+        receipt: dict[str, Any] = {"run_id": run_id, "complete": False, "directory": destination}
+        try:
+            result = await asyncio.wait_for(environment.exec(
+                f"{self._node_prefix()} node --input-type=module -e {shlex.quote(script)} "
+                f"{shlex.quote(f'/tmp/hitch-state/runs/{run_id}')} {shlex.quote(destination)}"
+            ), timeout=5)
+            receipt["export_return_code"] = result.return_code
+        except (Exception, asyncio.CancelledError) as error:
+            # Preserve the original exception, not a best-effort export failure.
+            receipt["export_error"] = type(error).__name__
+        try:
+            (self.logs_dir / "hitch-interrupted-run.json").write_text(json.dumps(receipt) + "\n")
+        except OSError:
+            pass
+
+    async def _collect_run(
+        self, environment: BaseEnvironment, context: AgentContext, execution: ExecResult, *,
+        assigned_run_id: str, context_payload: dict[str, Any], parent_payload: dict[str, Any] | None,
+        prepared_phase: PreparedPhase | None,
+        workdir: str, proxy_health: str,
+    ) -> None:
+        run_id = assigned_run_id
+        trial_id, task_id, attempt = self._trial_identity()
         events = self._events(execution.stdout or "")
         observed_run_id = next((
             value
@@ -641,7 +1052,7 @@ class HitchHarborAgent(BaseAgent):
             for value in [event.get("run_id")]
             if isinstance(value, str) and re.fullmatch(r"run_[a-f0-9]{32}", value)
         ), None)
-        if observed_run_id:
+        if observed_run_id and prepared_phase is None:
             run_id = str(observed_run_id)
         result_path = f"/tmp/hitch-state/runs/{run_id}/result.json"
         quoted_result_path = shlex.quote(result_path)
@@ -664,7 +1075,7 @@ cat -- {quoted_result_path}
             "trial_id": trial_id,
             "completed_at": datetime.now(timezone.utc).isoformat(),
         }, separators=(",", ":"))
-        bundle_export = await environment.exec(
+        legacy_export = (
             f"""
 set -eu
 source_dir={shlex.quote(f'/tmp/hitch-state/runs/{run_id}')}
@@ -680,10 +1091,39 @@ rm -rf "$target_dir"
 mv "$stage_dir" "$target_dir"
 """.strip()
         )
+        if prepared_phase is None:
+            bundle_export = await environment.exec(legacy_export)
+        else:
+            export_input = {"sourceDirectory": f"/tmp/hitch-state/runs/{run_id}", "destinationDirectory": bundle_stage,
+                            "expected": {"run_id": run_id, "context": context_payload, "parent": parent_payload,
+                                         "revision_identity": self.revision_identity}}
+            # Completion is outside the bundle: adding even a marker inside a
+            # sealed bundle changes its indexed file set and invalidates it.
+            script = (
+                f"import {{copySealedPhaseRunBundle}} from 'file:///opt/hitch/{PHASE_EXPORT_MODULE}';"
+                "import {open,lstat,rename,writeFile} from 'node:fs/promises';"
+                "const input=JSON.parse(process.argv[1]);"
+                "const target='/logs/agent/hitch-run-bundle';"
+                "const lock=await open('/logs/agent/.hitch-phase-export.lock','wx',0o600);await lock.close();"
+                "try{await lstat(target);throw new Error('phase export target already exists')}catch(e){if(e.code!=='ENOENT')throw e;}"
+                "const index=await copySealedPhaseRunBundle(input);"
+                "await rename(input.destinationDirectory,target);"
+                "await writeFile('/logs/agent/hitch-phase.complete.json',JSON.stringify({schema_version:'1',"
+                "run_id:index.run_id,bundle_digest:index.bundle_digest,scope:'candidate-evidence-only'}),{flag:'wx',mode:0o600});"
+            )
+            bundle_export = await environment.exec(
+                f"{self._node_prefix()} node --input-type=module -e {shlex.quote(script)} {shlex.quote(json.dumps(export_input))}"
+            )
         hitch_result, result_error_code, result_error_message = self._parse_hitch_result(result_read, run_id)
+        if prepared_phase is None and hitch_result is not None and getattr(environment, "_hitch_benchmark", None):
+            from hitch_benchmark import export_final_response
+            await export_final_response(environment, hitch_result)
         primary_code: str | None = None
         primary_message: str | None = None
-        if execution.return_code != 0:
+        if prepared_phase is not None and observed_run_id and observed_run_id != assigned_run_id:
+            primary_code = "hitch_phase_run_identity_mismatch"
+            primary_message = "Hitch phase emitted a different run ID from its prepared tool binding"
+        elif execution.return_code != 0:
             primary_code = "hitch_process_failed"
             diagnostic = self._exec_diagnostic(execution)
             if hitch_result and isinstance(hitch_result.get("error"), dict):
@@ -711,6 +1151,13 @@ mv "$stage_dir" "$target_dir"
             primary_code = "hitch_result_artifact_copy_failed"
             primary_message = f"Hitch result artifact copy failed (run_id={run_id}, trial_id={trial_id})"
 
+        self._write_trusted_agent_outcome(
+            run_id=run_id,
+            hitch_result=hitch_result,
+            bundle_export=bundle_export,
+            reason_code=primary_code,
+        )
+
         context.metadata = {
             "candidate_id": self.candidate_id,
             "harness_ref": self.harness_ref,
@@ -728,6 +1175,11 @@ mv "$stage_dir" "$target_dir"
             "hitch_status": hitch_result.get("status") if hitch_result else None,
             "hitch_artifact_id": hitch_result.get("artifact_id") if hitch_result else None,
         }
+        if prepared_phase is not None:
+            context.metadata.update(hitch_context_kind="benchmark_phase", hitch_run_group_id=context_payload["run_group_id"],
+                                    hitch_phase_index=context_payload["phase_index"],
+                                    hitch_phase_bundle_exported=bundle_export.return_code == 0,
+                                    hitch_phase_completion="hitch-phase.complete.json")
         if self._artifact_manifest is not None:
             context.metadata["harness_artifact_transport"] = {
                 "artifact_id": self._artifact_manifest["artifact_id"],
@@ -945,6 +1397,37 @@ mv "$stage_dir" "$target_dir"
             cwd="/",
         )
 
+    def _write_trusted_agent_outcome(
+        self,
+        *,
+        run_id: str,
+        hitch_result: dict[str, Any] | None,
+        bundle_export: ExecResult,
+        reason_code: str | None,
+    ) -> None:
+        result_status = hitch_result.get("status") if hitch_result else "failed"
+        if result_status not in {"succeeded", "failed", "timed_out", "cancelled"}:
+            result_status = "failed"
+        bundle = "complete" if hitch_result is not None and bundle_export.return_code == 0 else (
+            "missing" if bundle_export.return_code != 0 else "invalid"
+        )
+        outcome = {
+            "schema_version": "1",
+            "run_id": run_id,
+            "status": result_status,
+            "candidate_bundle": bundle,
+            "submission_snapshot": "not-required",
+            "gradeability": "gradeable" if bundle == "complete" else "ungradeable",
+        }
+        if bundle != "complete":
+            outcome["reason_code"] = reason_code or "candidate_evidence_unavailable"
+        self.logs_dir.mkdir(parents=True, exist_ok=True)
+        target = self.logs_dir / HITCH_AGENT_OUTCOME_NAME
+        temporary = target.with_name(f".{target.name}.{uuid.uuid4().hex}.tmp")
+        temporary.write_text(json.dumps(outcome, separators=(",", ":"), sort_keys=True) + "\n", encoding="utf-8")
+        temporary.chmod(0o600)
+        temporary.replace(target)
+
     def _trial_identity(self) -> tuple[str, str, int]:
         """Read Harbor's stable trial/task identity from the persisted trial state."""
         trial_dir = self.logs_dir.parent if self.logs_dir.name == "agent" else self.logs_dir
@@ -1024,43 +1507,158 @@ process.stdout.write('sha256:' + hash.digest('hex'));
         return "sha256:" + hashlib.sha256(b"workspace-unavailable").hexdigest()
 
     async def _ensure_node(self, environment: BaseEnvironment) -> None:
-        probe = await environment.exec(
-            f"node -e 'process.exit(process.version === \"{self.required_node_version}\" ? 0 : 1)'"
+        """Select a compatible system Node or an authenticated offline runtime.
+
+        No network, package manager, shell profile, or existing system binary
+        is modified here. The archive travels inside the job-pinned artifact,
+        including on remote workers and reruns.
+        """
+        platform = str((self._artifact_manifest or {}).get("platform", ""))
+        self._node_bin_directory = None
+        check = (
+            f"process.exit(process.version === {json.dumps(self.required_node_version)} && "
+            f"process.platform + '-' + process.arch === {json.dumps(platform)} ? 0 : 1)"
         )
-        if probe.return_code == 0:
-            return
-        prerequisites = """
-set -eu
-if command -v curl >/dev/null 2>&1; then exit 0; fi
-if command -v apt-get >/dev/null 2>&1; then
-  apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y curl ca-certificates
-elif command -v apk >/dev/null 2>&1; then
-  apk add --no-cache curl ca-certificates bash
-elif command -v dnf >/dev/null 2>&1; then
-  dnf install -y curl ca-certificates
-elif command -v yum >/dev/null 2>&1; then
-  yum install -y curl ca-certificates
-else
-  echo 'Hitch could not install the pinned Node.js runtime in this task image' >&2
-  exit 1
-fi
-"""
-        await self._exec(environment, prerequisites, user=0)
-        install = f"""
-set -eu
-export NVM_DIR=/opt/hitch-node
-mkdir -p "$NVM_DIR"
-curl -fsSL https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.3/install.sh | bash
-. "$NVM_DIR/nvm.sh"
-nvm install {self.required_node_version.removeprefix("v")}
-nvm alias default {self.required_node_version.removeprefix("v")}
-node -e 'process.exit(process.version === "{self.required_node_version}" ? 0 : 1)'
-"""
-        await self._exec(environment, install, user=0)
+        staging: str | None = None
+        try:
+            probe = await asyncio.wait_for(environment.exec(f"node -e {shlex.quote(check)}"), timeout=30)
+            if probe.return_code == 0:
+                self._node_bin_directory = None
+                self._record_node_runtime({"source": "system", "node_version": self.required_node_version, "platform": platform})
+                return
+            archive, runtime = self._offline_node_runtime()
+            native = await self._node_setup_exec(environment, """set -eu
+test "$(uname -s)" = Linux
+case "$(uname -m)" in
+  x86_64|amd64) echo linux-x64 ;;
+  aarch64|arm64) echo linux-arm64 ;;
+  *) exit 1 ;;
+esac""", "hitch_node_runtime_incompatible")
+            if (native.stdout or "").strip() != runtime["platform"]:
+                raise self._node_runtime_error("hitch_node_runtime_incompatible", "offline Node architecture does not match the task container")
+            libc = await self._node_setup_exec(environment, "getconf GNU_LIBC_VERSION", "hitch_node_runtime_incompatible")
+            if not re.fullmatch(r"glibc \d+\.\d+", (libc.stdout or "").strip()):
+                raise self._node_runtime_error("hitch_node_runtime_incompatible", "offline Node requires a glibc task image; musl is not supported")
+            await self._node_setup_exec(
+                environment, "command -v sha256sum && command -v tar && command -v gzip", "hitch_node_runtime_prerequisite_missing",
+            )
+            # Unique per setup; never extract over a system Node or an old,
+            # partially installed runtime. The ready directory appears only
+            # after checksum, extraction and executable compatibility checks.
+            target = f"/opt/hitch-node-runtime-{uuid.uuid4().hex}"
+            await self._node_setup_exec(environment, f"mkdir -m 755 {target}", "hitch_node_runtime_install_failed")
+            staging = target
+            await asyncio.wait_for(environment.upload_file(archive, f"{staging}/node-runtime.tar.gz"), timeout=120)
+            checksum = runtime["archive_sha256"].removeprefix("sha256:")
+            await self._node_setup_exec(
+                environment,
+                f"cd {staging} && printf '%s\\n' '{checksum}  node-runtime.tar.gz' | sha256sum -c -",
+                "hitch_node_runtime_integrity_mismatch",
+            )
+            await self._node_setup_exec(
+                environment,
+                f"umask 022; mkdir -m 755 {staging}/unpacked && tar --no-same-owner -xzf {staging}/node-runtime.tar.gz -C {staging}/unpacked",
+                "hitch_node_runtime_install_failed",
+            )
+            await self._node_setup_exec(
+                environment, f"{staging}/unpacked/bin/node -e {shlex.quote(check)}", "hitch_node_runtime_incompatible",
+            )
+            await self._node_setup_exec(
+                environment, f"mv {staging}/unpacked {staging}/ready && rm {staging}/node-runtime.tar.gz", "hitch_node_runtime_install_failed",
+            )
+            self._node_bin_directory = f"{staging}/ready/bin"
+            self._record_node_runtime({"source": "offline-artifact", **runtime, "bin_directory": self._node_bin_directory})
+            staging = None
+        except Exception as error:
+            failure = error if isinstance(error, HitchBridgeError) else self._node_runtime_error(
+                "hitch_node_runtime_setup_failed", f"{type(error).__name__}: {error}",
+            )
+            self._record_node_runtime({"source": "failed", **failure.evidence})
+            try:
+                await self._write_bridge_error(environment, failure.evidence)
+            except Exception:
+                pass  # Preserve the original Node failure even if log export fails.
+            if failure is error:
+                raise
+            raise failure from error
+        finally:
+            if staging is not None:
+                # Only the random directory created by this setup is removed.
+                try:
+                    await asyncio.wait_for(environment.exec(f"rm -rf -- {staging}", user=0), timeout=30)
+                except Exception:
+                    pass
 
-    @staticmethod
-    def _node_prefix() -> str:
-        return "if [ -s /opt/hitch-node/nvm.sh ]; then export NVM_DIR=/opt/hitch-node; . /opt/hitch-node/nvm.sh; fi;"
+    def _offline_node_runtime(self) -> tuple[Path, dict[str, Any]]:
+        directory = self._artifact_host_directory
+        if directory is None or not (directory / ".hitch-node-runtime").exists():
+            raise self._node_runtime_error(
+                "hitch_node_runtime_missing",
+                "task has no matching Node and the pinned artifact has no offline runtime; prepare a new eval with the updated controller (no online fallback)",
+            )
+        try:
+            # Authenticate the runtime metadata against the job's CONTENT pin,
+            # not a self-reported checksum in a mutable sidecar. This is needed
+            # before Node can run Hitch's normal in-container artifact check.
+            captured = {".hitch-node-runtime/node-runtime.json": b""}
+            if artifact_directory_integrity(directory, captured) != (self.harness_artifact or {}).get("artifact_integrity"):
+                raise RuntimeError("job-pinned harness content digest mismatch before Node bootstrap")
+            bundle = directory / ".hitch-node-runtime"
+            if bundle.is_symlink() or not bundle.is_dir():
+                raise RuntimeError("offline Node bundle must be a regular directory")
+            manifest_path = bundle / "node-runtime.json"
+            self._assert_regular_host_file(manifest_path, "offline Node manifest", 16_384)
+            # Parse exactly the bytes hashed above, not a second sidecar read
+            # which could race a cache mutation after content authentication.
+            runtime = json.loads(captured[".hitch-node-runtime/node-runtime.json"].decode("utf-8"))
+            fields = {"schema_version", "recipe_version", "runtime_id", "node_version", "platform", "libc", "builder_image_id", "archive_sha256", "archive_bytes"}
+            if not isinstance(runtime, dict) or set(runtime) != fields:
+                raise RuntimeError("offline Node manifest fields are invalid")
+            payload = {key: value for key, value in runtime.items() if key != "runtime_id"}
+            identity = "sha256:" + hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
+            if runtime["schema_version"] != "1" or runtime["recipe_version"] != "1" or runtime["runtime_id"] != identity:
+                raise RuntimeError("offline Node runtime identity is invalid")
+            if runtime["node_version"] != self.required_node_version or runtime["platform"] != (self._artifact_manifest or {}).get("platform") or runtime["libc"] != "glibc":
+                raise RuntimeError("offline Node manifest does not match the job runtime contract")
+            for field in ("archive_sha256", "builder_image_id"):
+                if not isinstance(runtime[field], str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", runtime[field]):
+                    raise RuntimeError(f"offline Node {field} is invalid")
+            archive = bundle / "node-runtime.tar.gz"
+            info = self._assert_regular_host_file(archive, "offline Node archive", 128 * 1024 * 1024)
+            if type(runtime["archive_bytes"]) is not int or info.st_size != runtime["archive_bytes"] or info.st_size <= 0:
+                raise RuntimeError("offline Node archive size mismatch")
+            archive_digest = hashlib.sha256()
+            with archive.open("rb") as handle:
+                for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+                    archive_digest.update(chunk)
+            if "sha256:" + archive_digest.hexdigest() != runtime["archive_sha256"]:
+                raise RuntimeError("offline Node archive checksum mismatch")
+            return archive, runtime
+        except Exception as error:
+            raise self._node_runtime_error("hitch_node_runtime_integrity_mismatch", str(error)) from error
+
+    async def _node_setup_exec(self, environment: BaseEnvironment, command: str, code: str) -> ExecResult:
+        result = await asyncio.wait_for(environment.exec(command, user=0), timeout=60)
+        if result.return_code != 0:
+            raise self._node_runtime_error(code, f"exit={result.return_code}: {self._exec_diagnostic(result)}")
+        return result
+
+    def _node_runtime_error(self, code: str, message: str) -> HitchBridgeError:
+        return HitchBridgeError(code, self._bounded_tail(message, 2048), {
+            "schema_version": "1", "code": code, "message": self._bounded_tail(message, 2048),
+            "eval_id": self.eval_id, "node_version": self.required_node_version,
+            "platform": (self._artifact_manifest or {}).get("platform"),
+            "artifact_id": (self.harness_artifact or {}).get("artifact_id"),
+        })
+
+    def _record_node_runtime(self, evidence: dict[str, Any]) -> None:
+        self.logs_dir.mkdir(parents=True, exist_ok=True)
+        (self.logs_dir / "hitch-node-runtime.json").write_text(
+            json.dumps({"schema_version": "1", **evidence}, indent=2, sort_keys=True) + "\n", encoding="utf-8",
+        )
+
+    def _node_prefix(self) -> str:
+        return f"export PATH={shlex.quote(self._node_bin_directory)}:\"$PATH\";" if self._node_bin_directory else ""
 
     @staticmethod
     def _events(output: str) -> list[dict[str, Any]]: