@neoline/hostbridge 2.0.4 → 2.0.5

package/src/server_control_mcp/tunnel_manager.py

@@ -0,0 +1,1245 @@
+"""Daemon-owned lifecycle management for persistent host tunnel providers."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import random
+import time
+import uuid
+from collections.abc import AsyncGenerator, Awaitable, Callable
+from contextlib import suppress
+from dataclasses import dataclass, field
+from ipaddress import ip_address, ip_network
+from typing import Protocol, cast
+
+from .hosts import ForwardingPolicy, HostConfig, HostRegistry
+from .transports.base import (
+    ByteStream,
+    ExecResult,
+    ShellAttachment,
+    ShellReadResult,
+    TransferResult,
+    TransportCapabilities,
+)
+
+
+class TunnelError(RuntimeError):
+    def __init__(self, code: str, message: str, *, retryable: bool = False) -> None:
+        super().__init__(message)
+        self.code = code
+        self.retryable = retryable
+
+
+class TunnelPolicyError(TunnelError):
+    def __init__(self, message: str) -> None:
+        super().__init__("denied", message)
+
+
+class TunnelCapacityError(TunnelError):
+    def __init__(self, message: str) -> None:
+        super().__init__("busy", message, retryable=True)
+
+
+class TunnelBackendError(TunnelError):
+    def __init__(self, message: str) -> None:
+        super().__init__("connection_lost", message, retryable=True)
+
+
+class TunnelTimeoutError(TunnelError):
+    def __init__(self, message: str) -> None:
+        super().__init__("timeout", message, retryable=True)
+
+
+class TunnelManagerClosed(TunnelError):
+    def __init__(self) -> None:
+        super().__init__("connection_lost", "host tunnel manager is closed")
+
+
+class TunnelProviderAbortError(RuntimeError):
+    """Raised when a provider violates the no-throw abort contract."""
+
+
+class TunnelUnsupportedError(TunnelError):
+    def __init__(self, message: str) -> None:
+        super().__init__("unsupported_transport", message)
+
+
+class TunnelProviderEndpoint(Protocol):
+    provider_stream_id: int
+
+    async def send(self, data: bytes) -> None: ...
+
+    async def send_eof(self) -> None: ...
+
+    async def receive(self, max_bytes: int = 64 * 1024) -> bytes | None: ...
+
+    async def close(self) -> None: ...
+
+    async def reset(self, reason: str) -> None: ...
+
+
+class TunnelProviderRuntime(Protocol):
+    provider_name: str
+
+    @property
+    def queued_bytes(self) -> int: ...
+
+    @property
+    def agent_pid(self) -> int | None: ...
+
+    async def start(self) -> None: ...
+
+    async def open_stream(
+        self,
+        host: str,
+        port: int,
+        policy: ForwardingPolicy,
+    ) -> TunnelProviderEndpoint: ...
+
+    async def wait_closed(self) -> BaseException | None: ...
+
+    async def close(self) -> None: ...
+
+    def abort(self) -> None: ...
+
+
+class HostOperationProvider(Protocol):
+    @property
+    def capabilities(self) -> TransportCapabilities: ...
+
+    async def exec(
+        self,
+        command: str,
+        *,
+        stdin: ByteStream | None,
+        timeout: float,
+        output_limit: int,
+    ) -> ExecResult: ...
+
+    async def upload(
+        self,
+        chunks: ByteStream,
+        remote_path: str,
+        *,
+        mode: int,
+        expected_size: int | None,
+        expected_sha256: str | None,
+    ) -> TransferResult: ...
+
+    def download(self, remote_path: str, *, chunk_size: int) -> AsyncGenerator[bytes, None]: ...
+
+    async def shell_write(self, shell_id: str, data: bytes) -> None: ...
+
+    async def shell_read(self, shell_id: str, *, timeout: float | None, max_bytes: int) -> ShellReadResult: ...
+
+    async def attach_shell(self, shell_id: str, stdin: ByteStream, *, max_bytes: int) -> ShellAttachment: ...
+
+    async def shell_close(self, shell_id: str) -> None: ...
+
+
+TunnelProviderFactory = Callable[[HostConfig, str, int], TunnelProviderRuntime]
+TunnelEventSink = Callable[[dict[str, object]], None]
+_StreamKey = tuple[str, int]
+_LOGGER = logging.getLogger(__name__)
+
+
+@dataclass(slots=True)
+class _TunnelMetrics:
+    bytes_sent: int = 0
+    bytes_received: int = 0
+    streams_opened: int = 0
+    streams_closed: int = 0
+    streams_reset: int = 0
+    stream_timeouts: int = 0
+    queue_high_water_bytes: int = 0
+    window_stalls: int = 0
+    protocol_failures: int = 0
+    last_open_latency_ms: float = 0.0
+    cleanup_errors: int = 0
+
+    def describe(self) -> dict[str, object]:
+        return {
+            "bytes_sent": self.bytes_sent,
+            "bytes_received": self.bytes_received,
+            "streams_opened": self.streams_opened,
+            "streams_closed": self.streams_closed,
+            "streams_reset": self.streams_reset,
+            "stream_timeouts": self.stream_timeouts,
+            "queue_high_water_bytes": self.queue_high_water_bytes,
+            "window_stalls": self.window_stalls,
+            "protocol_failures": self.protocol_failures,
+            "last_open_latency_ms": self.last_open_latency_ms,
+            "cleanup_errors": self.cleanup_errors,
+        }
+
+
+class TunnelStreamHandle:
+    def __init__(
+        self,
+        manager: HostTunnelManager,
+        *,
+        host_id: str,
+        client_connection_id: str,
+        client_stream_id: int,
+        runtime_generation: int,
+        endpoint: TunnelProviderEndpoint,
+        idle_timeout_seconds: float,
+        stream_deadline_seconds: float,
+        destination_host: str,
+        destination_port: int,
+        policy_rule: str,
+        opened_at: float,
+    ) -> None:
+        self._manager = manager
+        self.host_id = host_id
+        self.client_connection_id = client_connection_id
+        self.client_stream_id = client_stream_id
+        self.runtime_generation = runtime_generation
+        self._endpoint = endpoint
+        self.destination_host = destination_host
+        self.destination_port = destination_port
+        self.policy_rule = policy_rule
+        self.opened_at = opened_at
+        self.bytes_sent = 0
+        self.bytes_received = 0
+        self._local_eof = False
+        self._remote_eof = False
+        loop = asyncio.get_running_loop()
+        self._terminal: asyncio.Future[None] = loop.create_future()
+        self._idle_timeout_seconds = idle_timeout_seconds
+        self._last_activity = loop.time()
+        self._deadline = self._last_activity + stream_deadline_seconds
+        self._activity = asyncio.Event()
+        self._timeout_task = asyncio.create_task(
+            self._watch_timeouts(),
+            name=f"hostbridge-tunnel-stream-timeout-{host_id}-{runtime_generation}-{client_stream_id}",
+        )
+
+    @property
+    def provider_stream_id(self) -> int:
+        return self._endpoint.provider_stream_id
+
+    @property
+    def endpoint(self) -> TunnelProviderEndpoint:
+        return self._endpoint
+
+    @property
+    def closed(self) -> bool:
+        return self._terminal.done()
+
+    async def send(self, data: bytes) -> None:
+        if self.closed:
+            raise TunnelBackendError("tunnel stream is closed")
+        await self._endpoint.send(data)
+        if data:
+            self.bytes_sent += len(data)
+            self._manager._record_stream_bytes(self, sent=len(data))
+            self._record_activity()
+
+    async def send_eof(self) -> None:
+        if self.closed or self._local_eof:
+            return
+        await self._endpoint.send_eof()
+        self._local_eof = True
+        self._manager._record_stream_half_close(self, "to_destination")
+        self._record_activity()
+
+    async def receive(self, max_bytes: int = 64 * 1024) -> bytes | None:
+        if self.closed:
+            await self.wait_closed()
+            return None
+        if self._remote_eof:
+            return None
+        data = await self._endpoint.receive(max_bytes)
+        if data:
+            self.bytes_received += len(data)
+            self._manager._record_stream_bytes(self, received=len(data))
+            self._record_activity()
+        elif data is None:
+            self._remote_eof = True
+            self._manager._record_stream_half_close(self, "from_destination")
+            self._record_activity()
+        return data
+
+    async def close(self) -> None:
+        await self._manager._close_stream(self, reset_reason=None)
+
+    async def reset(self, reason: str = "stream reset") -> None:
+        await self._manager._close_stream(self, reset_reason=reason)
+
+    async def wait_closed(self) -> None:
+        await asyncio.shield(self._terminal)
+
+    def _finish(self, error: BaseException | None = None) -> None:
+        if self._terminal.done():
+            return
+        timeout_task = self._timeout_task
+        if timeout_task is not asyncio.current_task():
+            timeout_task.cancel()
+        self._activity.set()
+        if error is None:
+            self._terminal.set_result(None)
+        else:
+            self._terminal.set_exception(error)
+            self._terminal.exception()
+
+    def _record_activity(self) -> None:
+        self._last_activity = asyncio.get_running_loop().time()
+        self._activity.set()
+
+    async def _watch_timeouts(self) -> None:
+        loop = asyncio.get_running_loop()
+        try:
+            while not self.closed:
+                self._activity.clear()
+                now = loop.time()
+                idle_remaining = self._idle_timeout_seconds - (now - self._last_activity)
+                deadline_remaining = self._deadline - now
+                remaining = min(idle_remaining, deadline_remaining)
+                if remaining <= 0:
+                    reason = "deadline" if deadline_remaining <= idle_remaining else "idle"
+                    await self._manager._timeout_stream(self, reason)
+                    return
+                try:
+                    await asyncio.wait_for(self._activity.wait(), timeout=remaining)
+                except TimeoutError:
+                    continue
+        except asyncio.CancelledError:
+            return
+
+
+class HostRuntimeLease:
+    def __init__(
+        self,
+        manager: HostTunnelManager,
+        *,
+        host_id: str,
+        generation: int,
+        token: str,
+        capabilities: TransportCapabilities,
+    ) -> None:
+        self._manager = manager
+        self.host_id = host_id
+        self.generation = generation
+        self.capabilities = capabilities
+        self._token = token
+        self._closed = False
+        self._close_task: asyncio.Task[None] | None = None
+
+    @property
+    def closed(self) -> bool:
+        return self._closed
+
+    async def exec(
+        self,
+        command: str,
+        *,
+        stdin: ByteStream | None,
+        timeout: float,
+        output_limit: int,
+    ) -> ExecResult:
+        provider = await self._provider()
+        return await provider.exec(command, stdin=stdin, timeout=timeout, output_limit=output_limit)
+
+    async def upload(
+        self,
+        chunks: ByteStream,
+        remote_path: str,
+        *,
+        mode: int,
+        expected_size: int | None,
+        expected_sha256: str | None,
+    ) -> TransferResult:
+        provider = await self._provider()
+        return await provider.upload(
+            chunks,
+            remote_path,
+            mode=mode,
+            expected_size=expected_size,
+            expected_sha256=expected_sha256,
+        )
+
+    async def download(self, remote_path: str, *, chunk_size: int) -> AsyncGenerator[bytes, None]:
+        provider = await self._provider()
+        async for chunk in provider.download(remote_path, chunk_size=chunk_size):
+            yield chunk
+
+    async def shell_write(self, data: bytes) -> None:
+        provider = await self._provider()
+        await provider.shell_write(self._token, data)
+
+    async def shell_read(self, *, timeout: float | None, max_bytes: int) -> ShellReadResult:
+        provider = await self._provider()
+        return await provider.shell_read(self._token, timeout=timeout, max_bytes=max_bytes)
+
+    async def attach_shell(self, stdin: ByteStream, *, max_bytes: int) -> ShellAttachment:
+        provider = await self._provider()
+        return await provider.attach_shell(self._token, stdin, max_bytes=max_bytes)
+
+    async def close(self) -> None:
+        task = self._close_task
+        if task is None:
+            task = asyncio.create_task(
+                self._close_owned_shell(),
+                name=f"hostbridge-runtime-lease-close-{self.host_id}-{self.generation}",
+            )
+            task.add_done_callback(_consume_task_result)
+            self._close_task = task
+        await asyncio.shield(task)
+
+    async def _close_owned_shell(self) -> None:
+        try:
+            provider = await self._manager._lease_provider(self)
+            await provider.shell_close(self._token)
+        finally:
+            self._closed = True
+            await self._manager._release_lease(self)
+
+    async def _provider(self) -> HostOperationProvider:
+        if self._closed or self._close_task is not None:
+            raise TunnelBackendError("host runtime lease is closed")
+        return await self._manager._lease_provider(self)
+
+
+@dataclass(slots=True)
+class _HostRuntime:
+    host: HostConfig
+    state: str = "disconnected"
+    generation: int = 0
+    provider: TunnelProviderRuntime | None = None
+    connect_task: asyncio.Task[TunnelProviderRuntime] | None = None
+    monitor_task: asyncio.Task[None] | None = None
+    idle_task: asyncio.Task[None] | None = None
+    streams: dict[_StreamKey, TunnelStreamHandle] = field(default_factory=dict)
+    reservations: set[_StreamKey] = field(default_factory=set)
+    leases: dict[str, int] = field(default_factory=dict)
+    last_error: str | None = None
+    connected_at: float | None = None
+    consecutive_failures: int = 0
+    reconnect_count: int = 0
+    reconnect_delay_seconds: float = 0.0
+    metrics: _TunnelMetrics = field(default_factory=_TunnelMetrics)
+    sampled_window_stalls: int = 0
+    sampled_protocol_failures: int = 0
+    lock: asyncio.Lock = field(default_factory=asyncio.Lock)
+
+
+class HostTunnelManager:
+    def __init__(
+        self,
+        hosts: HostRegistry,
+        provider_factory: TunnelProviderFactory,
+        *,
+        sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
+        jitter: Callable[[float], float] | None = None,
+        event_sink: TunnelEventSink | None = None,
+        cleanup_timeout_seconds: float = 5,
+    ) -> None:
+        if cleanup_timeout_seconds <= 0:
+            raise ValueError("cleanup_timeout_seconds must be positive")
+        self._hosts = hosts
+        self._provider_factory = provider_factory
+        self._sleep = sleep
+        self._jitter = jitter or _default_reconnect_jitter
+        self._event_sink = event_sink
+        self._cleanup_timeout_seconds = float(cleanup_timeout_seconds)
+        self._runtimes: dict[str, _HostRuntime] = {}
+        self._provider_cleanup_tasks: set[asyncio.Task[None]] = set()
+        self._closed = False
+        self._close_task: asyncio.Task[None] | None = None
+
+    @property
+    def hosts(self) -> HostRegistry:
+        return self._hosts
+
+    async def acquire(self, host_id: str) -> HostRuntimeLease:
+        if self._closed:
+            raise TunnelManagerClosed()
+        host = self._hosts.get(host_id)
+        provider_name = _provider_name(host)
+        runtime = self._runtimes.setdefault(host.id, _HostRuntime(host))
+        while True:
+            provider, generation = await self._ensure_provider(runtime, provider_name)
+            _host_operation_provider(provider)
+            async with runtime.lock:
+                if self._closed:
+                    raise TunnelManagerClosed()
+                if runtime.state != "ready" or runtime.provider is not provider or runtime.generation != generation:
+                    continue
+                self._cancel_idle_locked(runtime)
+                token = uuid.uuid4().hex
+                runtime.leases[token] = generation
+                return HostRuntimeLease(
+                    self,
+                    host_id=host.id,
+                    generation=generation,
+                    token=token,
+                    capabilities=_host_operation_provider(provider).capabilities,
+                )
+
+    async def open_stream(
+        self,
+        host_id: str,
+        client_connection_id: str,
+        client_stream_id: int,
+        destination_host: str,
+        destination_port: int,
+    ) -> TunnelStreamHandle:
+        if self._closed:
+            raise TunnelManagerClosed()
+        if not isinstance(client_connection_id, str) or not client_connection_id:
+            raise ValueError("client_connection_id must be a non-empty string")
+        if not isinstance(client_stream_id, int) or isinstance(client_stream_id, bool) or client_stream_id <= 0:
+            raise ValueError("client_stream_id must be a positive integer")
+        host = self._hosts.get(host_id)
+        provider_name = _provider_name(host)
+        policy_rule = _authorize_destination(host.forwarding, provider_name, destination_host, destination_port)
+        runtime = self._runtimes.setdefault(host.id, _HostRuntime(host))
+        key = (client_connection_id, client_stream_id)
+        open_started = time.monotonic()
+
+        while True:
+            provider, generation = await self._ensure_provider(runtime, provider_name)
+            async with runtime.lock:
+                if self._closed:
+                    raise TunnelManagerClosed()
+                if runtime.state != "ready" or runtime.provider is not provider or runtime.generation != generation:
+                    continue
+                self._cancel_idle_locked(runtime)
+                if key in runtime.streams or key in runtime.reservations:
+                    raise TunnelCapacityError("client stream already exists")
+                policy = runtime.host.forwarding
+                if len(runtime.streams) + len(runtime.reservations) >= policy.max_streams:
+                    raise TunnelCapacityError(f"host tunnel stream limit is {policy.max_streams}")
+                runtime.reservations.add(key)
+                break
+
+        try:
+            endpoint = await provider.open_stream(destination_host, destination_port, runtime.host.forwarding)
+        except asyncio.CancelledError:
+            await self._release_reservation(runtime, key, generation)
+            raise
+        except TunnelError:
+            await self._release_reservation(runtime, key, generation)
+            raise
+        except Exception as exc:
+            await self._release_reservation(runtime, key, generation)
+            raise TunnelBackendError(f"failed to open destination stream: {exc}") from exc
+
+        stale = False
+        async with runtime.lock:
+            runtime.reservations.discard(key)
+            if (
+                self._closed
+                or runtime.state != "ready"
+                or runtime.provider is not provider
+                or runtime.generation != generation
+            ):
+                stale = True
+            else:
+                handle = TunnelStreamHandle(
+                    self,
+                    host_id=host.id,
+                    client_connection_id=client_connection_id,
+                    client_stream_id=client_stream_id,
+                    runtime_generation=generation,
+                    endpoint=endpoint,
+                    idle_timeout_seconds=policy.idle_timeout_seconds,
+                    stream_deadline_seconds=policy.stream_deadline_seconds,
+                    destination_host=destination_host,
+                    destination_port=destination_port,
+                    policy_rule=policy_rule,
+                    opened_at=time.monotonic(),
+                )
+                runtime.streams[key] = handle
+        if stale:
+            await self._cleanup_endpoint(endpoint, reset_reason="backend generation changed while opening stream")
+            if self._closed:
+                raise TunnelManagerClosed()
+            raise TunnelBackendError("backend generation changed while opening stream")
+        open_latency_ms = (time.monotonic() - open_started) * 1000
+        runtime.metrics.streams_opened += 1
+        runtime.metrics.last_open_latency_ms = open_latency_ms
+        self._sample_provider_metrics(runtime)
+        self._emit(
+            "tunnel.stream.opened",
+            runtime,
+            stream_id=client_stream_id,
+            provider_stream_id=handle.provider_stream_id,
+            destination_host=destination_host,
+            destination_port=destination_port,
+            policy_rule=policy_rule,
+            open_latency_ms=open_latency_ms,
+        )
+        return handle
+
+    def health(self, host_id: str) -> dict[str, object]:
+        host = self._hosts.get(host_id)
+        runtime = self._runtimes.get(host.id)
+        if runtime is None:
+            return {
+                "host_id": host.id,
+                "enabled": host.forwarding.enabled,
+                "state": "disconnected",
+                "provider": _provider_name(host),
+                "generation": 0,
+                "stream_count": 0,
+                "lease_count": 0,
+                "queue_bytes": 0,
+                "last_error": None,
+                "connected_at": None,
+                "agent_pid": None,
+                "reconnect_count": 0,
+                "reconnect_delay_seconds": 0.0,
+                "metrics": _TunnelMetrics().describe(),
+            }
+        provider = runtime.provider
+        self._sample_provider_metrics(runtime)
+        return {
+            "host_id": host.id,
+            "enabled": host.forwarding.enabled,
+            "state": runtime.state,
+            "provider": _provider_name(host),
+            "generation": runtime.generation,
+            "stream_count": len(runtime.streams),
+            "lease_count": self._lease_count(runtime, runtime.generation),
+            "queue_bytes": 0 if provider is None else getattr(provider, "queued_bytes", 0),
+            "last_error": runtime.last_error,
+            "connected_at": runtime.connected_at,
+            "agent_pid": None if provider is None else getattr(provider, "agent_pid", None),
+            "reconnect_count": runtime.reconnect_count,
+            "reconnect_delay_seconds": runtime.reconnect_delay_seconds,
+            "metrics": runtime.metrics.describe(),
+        }
+
+    async def close(self) -> None:
+        task = self._close_task
+        if task is None:
+            self._closed = True
+            task = asyncio.create_task(
+                self._close_owned_resources(),
+                name="hostbridge-tunnel-manager-close",
+            )
+            task.add_done_callback(_consume_task_result)
+            self._close_task = task
+        await asyncio.shield(task)
+
+    async def _close_owned_resources(self) -> None:
+        tasks: list[asyncio.Task[object]] = []
+        providers: list[tuple[_HostRuntime, TunnelProviderRuntime]] = []
+        handles: list[TunnelStreamHandle] = []
+        runtimes = list(self._runtimes.values())
+        for runtime in runtimes:
+            async with runtime.lock:
+                monitor_is_cleaning_failure = runtime.state == "failed"
+                runtime.state = "draining"
+                if runtime.connect_task is not None:
+                    runtime.connect_task.cancel()
+                    tasks.append(runtime.connect_task)
+                    runtime.connect_task = None
+                if runtime.monitor_task is not None:
+                    if not monitor_is_cleaning_failure:
+                        runtime.monitor_task.cancel()
+                    tasks.append(runtime.monitor_task)
+                    runtime.monitor_task = None
+                if runtime.idle_task is not None:
+                    runtime.idle_task.cancel()
+                    tasks.append(runtime.idle_task)
+                    runtime.idle_task = None
+                if runtime.provider is not None:
+                    providers.append((runtime, runtime.provider))
+                    runtime.provider = None
+                handles.extend(runtime.streams.values())
+                runtime.streams.clear()
+                runtime.reservations.clear()
+                runtime.leases.clear()
+        error = TunnelManagerClosed()
+        await asyncio.gather(
+            *(self._finalize_detached_stream(self._runtimes[handle.host_id], handle, error) for handle in handles),
+            return_exceptions=True,
+        )
+        cleanup_results = await asyncio.gather(
+            *(self._close_provider(runtime, provider) for runtime, provider in providers),
+        )
+        cleanup_by_host = {
+            runtime.host.id: (cleanup_error, cleanup_duration_ms)
+            for (runtime, _provider), (cleanup_error, cleanup_duration_ms) in zip(
+                providers,
+                cleanup_results,
+                strict=True,
+            )
+        }
+        if tasks:
+            await asyncio.gather(*tasks, return_exceptions=True)
+        for runtime in runtimes:
+            runtime.state = "closed"
+            runtime.connected_at = None
+            cleanup_error, cleanup_duration_ms = cleanup_by_host.get(runtime.host.id, (None, 0.0))
+            self._emit(
+                "tunnel.runtime.closed",
+                runtime,
+                cleanup_error=cleanup_error,
+                cleanup_duration_ms=cleanup_duration_ms,
+            )
+
+    async def _ensure_provider(
+        self,
+        runtime: _HostRuntime,
+        provider_name: str,
+    ) -> tuple[TunnelProviderRuntime, int]:
+        async with runtime.lock:
+            if self._closed:
+                raise TunnelManagerClosed()
+            self._cancel_idle_locked(runtime)
+            if runtime.state == "ready" and runtime.provider is not None:
+                return runtime.provider, runtime.generation
+            task = runtime.connect_task
+            if task is None:
+                runtime.generation += 1
+                generation = runtime.generation
+                reconnect_delay = runtime.reconnect_delay_seconds
+                runtime.state = "connecting"
+                task = asyncio.create_task(
+                    self._connect_runtime(runtime, provider_name, generation, reconnect_delay),
+                    name=f"hostbridge-tunnel-connect-{runtime.host.id}-{generation}",
+                )
+                task.add_done_callback(_consume_provider_task_result)
+                runtime.connect_task = task
+                self._emit("tunnel.runtime.connecting", runtime)
+        provider = await asyncio.shield(task)
+        return provider, runtime.generation
+
+    async def _connect_runtime(
+        self,
+        runtime: _HostRuntime,
+        provider_name: str,
+        generation: int,
+        reconnect_delay: float,
+    ) -> TunnelProviderRuntime:
+        provider: TunnelProviderRuntime | None = None
+        try:
+            if reconnect_delay > 0:
+                await self._sleep(reconnect_delay)
+            provider = self._provider_factory(runtime.host, provider_name, generation)
+            await provider.start()
+            async with runtime.lock:
+                if self._closed or runtime.generation != generation:
+                    raise TunnelManagerClosed()
+                runtime.provider = provider
+                runtime.sampled_window_stalls = 0
+                runtime.sampled_protocol_failures = 0
+                runtime.state = "ready"
+                runtime.last_error = None
+                runtime.connected_at = time.monotonic()
+                runtime.consecutive_failures = 0
+                runtime.reconnect_delay_seconds = 0.0
+                runtime.connect_task = None
+                runtime.monitor_task = asyncio.create_task(
+                    self._monitor_provider(runtime, provider, generation),
+                    name=f"hostbridge-tunnel-monitor-{runtime.host.id}-{generation}",
+                )
+                self._emit("tunnel.runtime.ready", runtime)
+            return provider
+        except asyncio.CancelledError:
+            if provider is not None:
+                await self._close_provider(runtime, provider)
+            raise
+        except TunnelManagerClosed:
+            if provider is not None:
+                await self._close_provider(runtime, provider)
+            raise
+        except Exception as exc:
+            if provider is not None:
+                await self._close_provider(runtime, provider)
+            error = TunnelBackendError(f"failed to connect host tunnel: {exc}")
+            async with runtime.lock:
+                if runtime.generation == generation:
+                    runtime.state = "failed"
+                    runtime.last_error = str(error)
+                    runtime.connect_task = None
+                    runtime.provider = None
+                    runtime.connected_at = None
+                    self._record_runtime_failure(runtime)
+                    self._emit("tunnel.runtime.failed", runtime, error=str(error))
+            raise error from exc
+
+    async def _monitor_provider(
+        self,
+        runtime: _HostRuntime,
+        provider: TunnelProviderRuntime,
+        generation: int,
+    ) -> None:
+        monitor_task = asyncio.current_task()
+        try:
+            try:
+                cause = await provider.wait_closed()
+            except asyncio.CancelledError:
+                return
+            except Exception as exc:
+                cause = exc
+            error = TunnelBackendError(str(cause) if cause is not None else "backend connection closed")
+            async with runtime.lock:
+                if (
+                    self._closed
+                    or runtime.generation != generation
+                    or runtime.provider is not provider
+                    or runtime.state != "ready"
+                ):
+                    return
+                runtime.state = "failed"
+                self._sample_provider_metrics(runtime)
+                runtime.provider = None
+                runtime.last_error = str(error)
+                runtime.connected_at = None
+                self._record_runtime_failure(runtime)
+                self._emit("tunnel.runtime.failed", runtime, error=str(error))
+                handles = list(runtime.streams.values())
+                runtime.streams.clear()
+                runtime.reservations.clear()
+                self._cancel_idle_locked(runtime)
+            await asyncio.gather(
+                *(self._finalize_detached_stream(runtime, handle, error) for handle in handles),
+                return_exceptions=True,
+            )
+            await self._close_provider(runtime, provider)
+        finally:
+            async with runtime.lock:
+                if runtime.monitor_task is monitor_task:
+                    runtime.monitor_task = None
+
+    def _record_runtime_failure(self, runtime: _HostRuntime) -> None:
+        runtime.consecutive_failures += 1
+        runtime.reconnect_count += 1
+        base_delay = min(10.0, 0.25 * (2 ** (runtime.consecutive_failures - 1)))
+        runtime.reconnect_delay_seconds = min(10.0, max(0.25, self._jitter(base_delay)))
+
+    async def _close_stream(
+        self,
+        handle: TunnelStreamHandle,
+        *,
+        reset_reason: str | None,
+        terminal_error: BaseException | None = None,
+    ) -> None:
+        runtime = self._runtimes.get(handle.host_id)
+        if runtime is None:
+            handle._finish()
+            return
+        key = (handle.client_connection_id, handle.client_stream_id)
+        owns_cleanup = False
+        async with runtime.lock:
+            if runtime.streams.get(key) is handle:
+                del runtime.streams[key]
+                owns_cleanup = True
+        if not owns_cleanup:
+            return
+        cleanup_task = asyncio.create_task(
+            self._finalize_owned_stream(
+                runtime,
+                handle,
+                reset_reason=reset_reason,
+                terminal_error=terminal_error,
+            ),
+            name=f"hostbridge-tunnel-stream-close-{handle.host_id}-{handle.client_stream_id}",
+        )
+        waiter = asyncio.gather(cleanup_task, return_exceptions=True)
+        cancellation_requested = False
+        while True:
+            try:
+                results = await asyncio.shield(waiter)
+                break
+            except asyncio.CancelledError:
+                cancellation_requested = True
+        result = results[0]
+        if isinstance(result, BaseException):
+            raise result
+        if cancellation_requested:
+            raise asyncio.CancelledError
+
+    async def _finalize_owned_stream(
+        self,
+        runtime: _HostRuntime,
+        handle: TunnelStreamHandle,
+        *,
+        reset_reason: str | None,
+        terminal_error: BaseException | None,
+    ) -> None:
+        cleanup_error: str | None = None
+        cleanup_started = time.monotonic()
+        cleanup_error, _cleanup_duration_ms = await self._cleanup_endpoint(
+            handle.endpoint,
+            reset_reason=reset_reason,
+        )
+        if cleanup_error is not None:
+            runtime.metrics.cleanup_errors += 1
+        handle._finish(terminal_error)
+        duration_ms = (time.monotonic() - handle.opened_at) * 1000
+        if reset_reason is None:
+            runtime.metrics.streams_closed += 1
+            event = "tunnel.stream.closed"
+        else:
+            runtime.metrics.streams_reset += 1
+            if isinstance(terminal_error, TunnelTimeoutError):
+                runtime.metrics.stream_timeouts += 1
+            event = "tunnel.stream.reset"
+        self._sample_provider_metrics(runtime)
+        self._emit(
+            event,
+            runtime,
+            stream_id=handle.client_stream_id,
+            provider_stream_id=handle.provider_stream_id,
+            destination_host=handle.destination_host,
+            destination_port=handle.destination_port,
+            policy_rule=handle.policy_rule,
+            duration_ms=duration_ms,
+            bytes_sent=handle.bytes_sent,
+            bytes_received=handle.bytes_received,
+            reason=reset_reason,
+            cleanup_duration_ms=(time.monotonic() - cleanup_started) * 1000,
+            cleanup_error=cleanup_error,
+            error_code=None if terminal_error is None else getattr(terminal_error, "code", "connection_lost"),
+        )
+        async with runtime.lock:
+            if runtime.state == "ready" and not self._runtime_in_use(runtime, runtime.generation):
+                self._schedule_idle_locked(runtime)
+
+    async def _timeout_stream(self, handle: TunnelStreamHandle, reason: str) -> None:
+        message = "tunnel stream deadline exceeded" if reason == "deadline" else "tunnel stream idle timeout"
+        await self._close_stream(
+            handle,
+            reset_reason=message,
+            terminal_error=TunnelTimeoutError(message),
+        )
+
+    async def _finalize_detached_stream(
+        self,
+        runtime: _HostRuntime,
+        handle: TunnelStreamHandle,
+        error: TunnelError,
+    ) -> None:
+        cleanup_started = time.monotonic()
+        cleanup_error, _cleanup_duration_ms = await self._cleanup_endpoint(
+            handle.endpoint,
+            reset_reason=str(error),
+        )
+        if cleanup_error is not None:
+            runtime.metrics.cleanup_errors += 1
+        handle._finish(error)
+        runtime.metrics.streams_reset += 1
+        self._sample_provider_metrics(runtime)
+        self._emit(
+            "tunnel.stream.reset",
+            runtime,
+            stream_id=handle.client_stream_id,
+            provider_stream_id=handle.provider_stream_id,
+            destination_host=handle.destination_host,
+            destination_port=handle.destination_port,
+            policy_rule=handle.policy_rule,
+            duration_ms=(time.monotonic() - handle.opened_at) * 1000,
+            bytes_sent=handle.bytes_sent,
+            bytes_received=handle.bytes_received,
+            reason=str(error),
+            error_code=error.code,
+            cleanup_duration_ms=(time.monotonic() - cleanup_started) * 1000,
+            cleanup_error=cleanup_error,
+        )
+
+    async def _release_reservation(self, runtime: _HostRuntime, key: _StreamKey, generation: int) -> None:
+        async with runtime.lock:
+            if runtime.generation != generation:
+                return
+            runtime.reservations.discard(key)
+            if runtime.state == "ready" and not self._runtime_in_use(runtime, generation):
+                self._schedule_idle_locked(runtime)
+
+    async def _lease_provider(self, lease: HostRuntimeLease) -> HostOperationProvider:
+        if self._closed:
+            raise TunnelManagerClosed()
+        runtime = self._runtimes.get(lease.host_id)
+        if runtime is None:
+            raise TunnelBackendError("host runtime is unavailable")
+        async with runtime.lock:
+            if (
+                runtime.state != "ready"
+                or runtime.generation != lease.generation
+                or runtime.leases.get(lease._token) != lease.generation
+                or runtime.provider is None
+            ):
+                raise TunnelBackendError("host runtime generation is no longer available")
+            return _host_operation_provider(runtime.provider)
+
+    async def _release_lease(self, lease: HostRuntimeLease) -> None:
+        runtime = self._runtimes.get(lease.host_id)
+        if runtime is None:
+            return
+        async with runtime.lock:
+            if runtime.leases.get(lease._token) != lease.generation:
+                return
+            del runtime.leases[lease._token]
+            if runtime.state == "ready" and not self._runtime_in_use(runtime, runtime.generation):
+                self._schedule_idle_locked(runtime)
+
+    def _schedule_idle_locked(self, runtime: _HostRuntime) -> None:
+        self._cancel_idle_locked(runtime)
+        runtime.idle_task = asyncio.create_task(
+            self._idle_close(runtime, runtime.generation),
+            name=f"hostbridge-tunnel-idle-{runtime.host.id}-{runtime.generation}",
+        )
+
+    def _cancel_idle_locked(self, runtime: _HostRuntime) -> None:
+        if runtime.idle_task is not None:
+            runtime.idle_task.cancel()
+            runtime.idle_task = None
+
+    async def _idle_close(self, runtime: _HostRuntime, generation: int) -> None:
+        try:
+            await asyncio.sleep(runtime.host.forwarding.idle_timeout_seconds)
+            async with runtime.lock:
+                if (
+                    self._closed
+                    or runtime.generation != generation
+                    or runtime.state != "ready"
+                    or runtime.streams
+                    or runtime.reservations
+                    or self._lease_count(runtime, generation)
+                    or runtime.provider is None
+                ):
+                    return
+                runtime.state = "draining"
+                provider = runtime.provider
+                runtime.provider = None
+                monitor = runtime.monitor_task
+                runtime.monitor_task = None
+                runtime.idle_task = None
+            if monitor is not None:
+                monitor.cancel()
+                await asyncio.gather(monitor, return_exceptions=True)
+            cleanup_error, cleanup_duration_ms = await self._close_provider(runtime, provider)
+            async with runtime.lock:
+                if runtime.generation == generation and runtime.state == "draining":
+                    runtime.state = "disconnected"
+                    runtime.connected_at = None
+                    self._emit(
+                        "tunnel.runtime.closed",
+                        runtime,
+                        reason="idle",
+                        cleanup_error=cleanup_error,
+                        cleanup_duration_ms=cleanup_duration_ms,
+                    )
+        except asyncio.CancelledError:
+            return
+
+    async def _close_provider(
+        self,
+        runtime: _HostRuntime,
+        provider: TunnelProviderRuntime,
+    ) -> tuple[str | None, float]:
+        started = time.monotonic()
+        cleanup_error: str | None = None
+        cleanup = asyncio.create_task(
+            provider.close(),
+            name=f"hostbridge-provider-cleanup-{runtime.host.id}",
+        )
+        try:
+            done, _pending = await asyncio.wait({cleanup}, timeout=self._cleanup_timeout_seconds)
+        except asyncio.CancelledError as cancellation:
+            abort_error = self._abort_provider_cleanup(provider, cleanup)
+            if abort_error is not None:
+                cancellation.add_note(f"provider abort failed: {abort_error}")
+            raise
+        if not done:
+            cleanup_error = "provider cleanup timed out"
+            runtime.metrics.cleanup_errors += 1
+            abort_error = self._abort_provider_cleanup(provider, cleanup)
+            if abort_error is not None:
+                raise TunnelProviderAbortError(f"provider abort failed: {abort_error}") from abort_error
+        else:
+            try:
+                cleanup.result()
+            except Exception as exc:
+                cleanup_error = str(exc)
+                runtime.metrics.cleanup_errors += 1
+        return cleanup_error, (time.monotonic() - started) * 1000
+
+    def _abort_provider_cleanup(
+        self,
+        provider: TunnelProviderRuntime,
+        cleanup: asyncio.Task[None],
+    ) -> Exception | None:
+        abort_error: Exception | None = None
+        try:
+            provider.abort()
+        except Exception as exc:
+            abort_error = exc
+        cleanup.cancel()
+        self._provider_cleanup_tasks.add(cleanup)
+        cleanup.add_done_callback(self._provider_cleanup_finished)
+        return abort_error
+
+    def _provider_cleanup_finished(self, task: asyncio.Task[None]) -> None:
+        self._provider_cleanup_tasks.discard(task)
+        _consume_task_result(task)
+
+    async def _cleanup_endpoint(
+        self,
+        endpoint: TunnelProviderEndpoint,
+        *,
+        reset_reason: str | None,
+    ) -> tuple[str | None, float]:
+        started = time.monotonic()
+        cleanup_error: str | None = None
+        try:
+            async with asyncio.timeout(self._cleanup_timeout_seconds):
+                if reset_reason is None:
+                    await endpoint.close()
+                else:
+                    await endpoint.reset(reset_reason)
+        except TimeoutError:
+            cleanup_error = "endpoint cleanup timed out"
+        except Exception as exc:
+            cleanup_error = str(exc)
+        return cleanup_error, (time.monotonic() - started) * 1000
+
+    def _record_stream_bytes(
+        self,
+        handle: TunnelStreamHandle,
+        *,
+        sent: int = 0,
+        received: int = 0,
+    ) -> None:
+        runtime = self._runtimes.get(handle.host_id)
+        if runtime is None or runtime.generation != handle.runtime_generation:
+            return
+        runtime.metrics.bytes_sent += sent
+        runtime.metrics.bytes_received += received
+        self._sample_provider_metrics(runtime)
+
+    def _record_stream_half_close(self, handle: TunnelStreamHandle, direction: str) -> None:
+        runtime = self._runtimes.get(handle.host_id)
+        if runtime is None or runtime.generation != handle.runtime_generation:
+            return
+        self._emit(
+            "tunnel.stream.half_closed",
+            runtime,
+            stream_id=handle.client_stream_id,
+            provider_stream_id=handle.provider_stream_id,
+            direction=direction,
+            bytes_sent=handle.bytes_sent,
+            bytes_received=handle.bytes_received,
+        )
+
+    def _sample_provider_metrics(self, runtime: _HostRuntime) -> None:
+        provider = runtime.provider
+        queued = 0 if provider is None else max(0, getattr(provider, "queued_bytes", 0))
+        runtime.metrics.queue_high_water_bytes = max(runtime.metrics.queue_high_water_bytes, queued)
+        if provider is None:
+            return
+        window_stalls = max(0, getattr(provider, "window_stalls", 0))
+        protocol_failures = max(0, getattr(provider, "protocol_failures", 0))
+        if window_stalls >= runtime.sampled_window_stalls:
+            runtime.metrics.window_stalls += window_stalls - runtime.sampled_window_stalls
+        else:
+            runtime.metrics.window_stalls += window_stalls
+        if protocol_failures >= runtime.sampled_protocol_failures:
+            runtime.metrics.protocol_failures += protocol_failures - runtime.sampled_protocol_failures
+        else:
+            runtime.metrics.protocol_failures += protocol_failures
+        runtime.sampled_window_stalls = window_stalls
+        runtime.sampled_protocol_failures = protocol_failures
+
+    @staticmethod
+    def _lease_count(runtime: _HostRuntime, generation: int) -> int:
+        return sum(lease_generation == generation for lease_generation in runtime.leases.values())
+
+    def _runtime_in_use(self, runtime: _HostRuntime, generation: int) -> bool:
+        return bool(runtime.streams or runtime.reservations or self._lease_count(runtime, generation))
+
+    def _emit(self, event: str, runtime: _HostRuntime, **fields: object) -> None:
+        record: dict[str, object] = {
+            "event": event,
+            "timestamp": time.time(),
+            "host_id": runtime.host.id,
+            "provider": _provider_name(runtime.host),
+            "generation": runtime.generation,
+            **fields,
+        }
+        try:
+            if self._event_sink is not None:
+                self._event_sink(record)
+            else:
+                _LOGGER.info("HostBridge tunnel event", extra={"hostbridge_event": record})
+        except Exception:
+            _LOGGER.exception("HostBridge tunnel event sink failed")
+
+
+def _provider_name(host: HostConfig) -> str:
+    if host.transport == "ssh":
+        return "native_ssh"
+    if host.transport == "pty":
+        return "pty_agent"
+    if host.ssh is not None and not host.login_steps and not host.ssh.extra_args:
+        return "native_ssh"
+    return "pty_agent"
+
+
+def _host_operation_provider(provider: TunnelProviderRuntime) -> HostOperationProvider:
+    required = ("exec", "upload", "download", "shell_write", "shell_read", "shell_close")
+    if not all(callable(getattr(provider, name, None)) for name in required):
+        raise TunnelUnsupportedError(f"provider {provider.provider_name!r} does not support host operations")
+    if not isinstance(getattr(provider, "capabilities", None), TransportCapabilities):
+        raise TunnelUnsupportedError(f"provider {provider.provider_name!r} does not report host capabilities")
+    return cast(HostOperationProvider, provider)
+
+
+def _authorize_destination(
+    policy: ForwardingPolicy,
+    provider_name: str,
+    destination_host: str,
+    destination_port: int,
+) -> str:
+    if provider_name not in {"native_ssh", "pty_agent"}:
+        raise TunnelPolicyError(f"unsupported tunnel provider {provider_name!r}")
+    if not policy.enabled:
+        raise TunnelPolicyError("TCP forwarding is disabled for this host")
+    if (
+        not isinstance(destination_port, int)
+        or isinstance(destination_port, bool)
+        or destination_port not in policy.allow_ports
+    ):
+        raise TunnelPolicyError(f"TCP forwarding port {destination_port!r} is not allowed")
+    if not isinstance(destination_host, str) or not destination_host or "\x00" in destination_host:
+        raise TunnelPolicyError("TCP forwarding destination is invalid")
+    try:
+        address = ip_address(destination_host)
+    except ValueError:
+        hostname = destination_host.rstrip(".").lower()
+        if hostname not in policy.allow_hostnames:
+            raise TunnelPolicyError(f"TCP forwarding hostname {destination_host!r} is not allowed") from None
+        return f"hostname:{hostname}"
+    networks = []
+    try:
+        networks = [ip_network(value, strict=True) for value in policy.allow_cidrs]
+    except ValueError as exc:
+        raise TunnelPolicyError(f"host has an invalid forwarding CIDR policy: {exc}") from exc
+    matched = next(
+        (network for network in networks if address.version == network.version and address in network),
+        None,
+    )
+    if matched is None:
+        raise TunnelPolicyError(f"TCP forwarding destination {destination_host!r} is not allowed")
+    return f"cidr:{matched}"
+
+
+def _default_reconnect_jitter(delay: float) -> float:
+    # Reconnect scheduling jitter is not security-sensitive randomness.
+    return random.uniform(delay, min(10.0, delay * 1.25))  # nosec B311
+
+
+def _consume_task_result(task: asyncio.Task[None]) -> None:
+    with suppress(BaseException):
+        task.result()
+
+
+def _consume_provider_task_result(task: asyncio.Task[TunnelProviderRuntime]) -> None:
+    with suppress(BaseException):
+        task.result()