@neoline/hostbridge 2.0.4 → 2.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -9
- package/docs/ARCHITECTURE.md +62 -0
- package/package.json +2 -1
- package/pyproject.toml +1 -1
- package/src/server_control_mcp/__init__.py +4 -3
- package/src/server_control_mcp/async_lifecycle.py +41 -0
- package/src/server_control_mcp/cli.py +1 -1
- package/src/server_control_mcp/client.py +302 -287
- package/src/server_control_mcp/config.py +2 -1
- package/src/server_control_mcp/daemon.py +233 -526
- package/src/server_control_mcp/doctor.py +34 -2
- package/src/server_control_mcp/hosts.py +136 -2
- package/src/server_control_mcp/mock_ssh.py +52 -14
- package/src/server_control_mcp/mock_ssh_exec.py +147 -58
- package/src/server_control_mcp/mock_ssh_session.py +3 -2
- package/src/server_control_mcp/mock_ssh_sftp.py +126 -28
- package/src/server_control_mcp/mock_ssh_tunnel.py +141 -444
- package/src/server_control_mcp/mux_connection.py +326 -0
- package/src/server_control_mcp/mux_daemon.py +186 -0
- package/src/server_control_mcp/mux_protocol.py +279 -0
- package/src/server_control_mcp/mux_records.py +71 -0
- package/src/server_control_mcp/mux_rpc.py +1779 -0
- package/src/server_control_mcp/mux_service.py +506 -0
- package/src/server_control_mcp/mux_stream.py +283 -0
- package/src/server_control_mcp/mux_sync_client.py +224 -0
- package/src/server_control_mcp/remote_agent_bundle.py +56 -0
- package/src/server_control_mcp/remote_mux_agent.py +769 -0
- package/src/server_control_mcp/runtime_services.py +862 -0
- package/src/server_control_mcp/server.py +33 -18
- package/src/server_control_mcp/services.py +2 -666
- package/src/server_control_mcp/transports/__init__.py +4 -8
- package/src/server_control_mcp/transports/base.py +60 -38
- package/src/server_control_mcp/transports/ssh.py +206 -259
- package/src/server_control_mcp/tunnel_manager.py +1245 -0
- package/src/server_control_mcp/tunnel_native_ssh.py +454 -0
- package/src/server_control_mcp/tunnel_providers.py +20 -0
- package/src/server_control_mcp/tunnel_pty_agent.py +909 -0
- package/src/server_control_mcp/protocol.py +0 -363
- package/src/server_control_mcp/remote_tunnel_agent.py +0 -143
- package/src/server_control_mcp/transports/pty.py +0 -515
|
@@ -1,309 +1,64 @@
|
|
|
1
|
+
"""AsyncSSH direct-tcpip adaptation for persistent binary mux streams."""
|
|
2
|
+
|
|
1
3
|
from __future__ import annotations
|
|
2
4
|
|
|
3
5
|
import asyncio
|
|
4
|
-
import base64
|
|
5
|
-
import binascii
|
|
6
6
|
import sys
|
|
7
|
-
import uuid
|
|
8
7
|
from collections import deque
|
|
8
|
+
from collections.abc import Awaitable, Callable
|
|
9
9
|
from contextlib import suppress
|
|
10
|
-
from dataclasses import dataclass
|
|
11
|
-
from enum import StrEnum
|
|
12
10
|
from typing import Protocol
|
|
13
11
|
|
|
14
12
|
import asyncssh
|
|
15
13
|
|
|
16
|
-
|
|
17
|
-
from .remote_tunnel_agent import build_remote_tunnel_command
|
|
18
|
-
|
|
19
|
-
TUNNEL_PROTOCOL_VERSION = 1
|
|
20
|
-
UPSTREAM_FRAME_BYTES = 2048
|
|
21
|
-
DOWNSTREAM_FRAME_BYTES = 16 * 1024
|
|
22
|
-
TUNNEL_READ_BYTES = 64 * 1024
|
|
23
|
-
TUNNEL_WRITE_BATCH_BYTES = 32 * 1024
|
|
24
|
-
TUNNEL_HIGH_WATER_BYTES = 128 * 1024
|
|
14
|
+
TUNNEL_HIGH_WATER_BYTES = 256 * 1024
|
|
25
15
|
TUNNEL_LOW_WATER_BYTES = 64 * 1024
|
|
26
|
-
TUNNEL_MAX_LINE_BYTES = 128 * 1024
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
class TunnelProtocolError(ValueError):
|
|
30
|
-
pass
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
class Direction(StrEnum):
|
|
34
|
-
UPSTREAM = "U"
|
|
35
|
-
DOWNSTREAM = "D"
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
class FrameKind(StrEnum):
|
|
39
|
-
READY = "READY"
|
|
40
|
-
DATA = "DATA"
|
|
41
|
-
EOF = "EOF"
|
|
42
|
-
ERROR = "ERROR"
|
|
43
|
-
CLOSED = "CLOSED"
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
@dataclass(frozen=True, slots=True)
|
|
47
|
-
class TunnelFrame:
|
|
48
|
-
direction: Direction
|
|
49
|
-
sequence: int
|
|
50
|
-
kind: FrameKind
|
|
51
|
-
payload: bytes = b""
|
|
52
|
-
status: int | None = None
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
class TunnelCodec:
|
|
56
|
-
def __init__(self, prefix: str, *, inbound_direction: Direction) -> None:
|
|
57
|
-
if not prefix or any(character in prefix for character in "\r\n:"):
|
|
58
|
-
raise ValueError("tunnel prefix must be non-empty and must not contain CR, LF, or colon")
|
|
59
|
-
try:
|
|
60
|
-
prefix_bytes = prefix.encode("ascii")
|
|
61
|
-
except UnicodeEncodeError as exc:
|
|
62
|
-
raise ValueError("tunnel prefix must be ASCII") from exc
|
|
63
|
-
self._prefix = prefix_bytes
|
|
64
|
-
self._marker = prefix_bytes + b":V"
|
|
65
|
-
self._inbound_direction = inbound_direction
|
|
66
|
-
self._next_sequence = 0
|
|
67
|
-
self._buffer = bytearray()
|
|
68
|
-
|
|
69
|
-
def encode(self, frame: TunnelFrame) -> bytes:
|
|
70
|
-
self._validate_frame(frame)
|
|
71
|
-
status = b"-" if frame.status is None else str(frame.status).encode("ascii")
|
|
72
|
-
payload = base64.b64encode(frame.payload)
|
|
73
|
-
return (
|
|
74
|
-
b":".join(
|
|
75
|
-
(
|
|
76
|
-
self._prefix,
|
|
77
|
-
f"V{TUNNEL_PROTOCOL_VERSION}".encode("ascii"),
|
|
78
|
-
frame.direction.value.encode("ascii"),
|
|
79
|
-
str(frame.sequence).encode("ascii"),
|
|
80
|
-
frame.kind.value.encode("ascii"),
|
|
81
|
-
status,
|
|
82
|
-
payload,
|
|
83
|
-
)
|
|
84
|
-
)
|
|
85
|
-
+ b"\n"
|
|
86
|
-
)
|
|
87
|
-
|
|
88
|
-
def feed(self, data: bytes) -> list[TunnelFrame]:
|
|
89
|
-
self._buffer.extend(data)
|
|
90
|
-
frames: list[TunnelFrame] = []
|
|
91
|
-
while b"\n" in self._buffer:
|
|
92
|
-
raw_line, _, remainder = self._buffer.partition(b"\n")
|
|
93
|
-
self._buffer = bytearray(remainder)
|
|
94
|
-
line = raw_line.rstrip(b"\r")
|
|
95
|
-
marker = line.find(self._marker)
|
|
96
|
-
if marker < 0:
|
|
97
|
-
continue
|
|
98
|
-
frames.append(self._decode_line(bytes(line[marker:])))
|
|
99
|
-
if len(self._buffer) > TUNNEL_MAX_LINE_BYTES:
|
|
100
|
-
raise TunnelProtocolError("tunnel frame exceeds maximum line size")
|
|
101
|
-
return frames
|
|
102
|
-
|
|
103
|
-
def _decode_line(self, line: bytes) -> TunnelFrame:
|
|
104
|
-
fields = line.split(b":", 6)
|
|
105
|
-
if len(fields) != 7:
|
|
106
|
-
raise TunnelProtocolError("malformed tunnel frame")
|
|
107
|
-
prefix, version, direction_raw, sequence_raw, kind_raw, status_raw, payload_raw = fields
|
|
108
|
-
if prefix != self._prefix:
|
|
109
|
-
raise TunnelProtocolError("tunnel frame prefix mismatch")
|
|
110
|
-
if version != f"V{TUNNEL_PROTOCOL_VERSION}".encode("ascii"):
|
|
111
|
-
raise TunnelProtocolError("unsupported tunnel protocol version")
|
|
112
|
-
try:
|
|
113
|
-
direction = Direction(direction_raw.decode("ascii"))
|
|
114
|
-
except (UnicodeDecodeError, ValueError) as exc:
|
|
115
|
-
raise TunnelProtocolError("invalid tunnel frame direction") from exc
|
|
116
|
-
if direction is not self._inbound_direction:
|
|
117
|
-
raise TunnelProtocolError("unexpected tunnel frame direction")
|
|
118
|
-
try:
|
|
119
|
-
sequence = int(sequence_raw)
|
|
120
|
-
except ValueError as exc:
|
|
121
|
-
raise TunnelProtocolError("invalid tunnel frame sequence") from exc
|
|
122
|
-
if sequence != self._next_sequence:
|
|
123
|
-
raise TunnelProtocolError(f"tunnel frame sequence expected {self._next_sequence}, received {sequence}")
|
|
124
|
-
try:
|
|
125
|
-
kind = FrameKind(kind_raw.decode("ascii"))
|
|
126
|
-
except (UnicodeDecodeError, ValueError) as exc:
|
|
127
|
-
raise TunnelProtocolError("invalid tunnel frame kind") from exc
|
|
128
|
-
if status_raw == b"-":
|
|
129
|
-
status = None
|
|
130
|
-
else:
|
|
131
|
-
try:
|
|
132
|
-
status = int(status_raw)
|
|
133
|
-
except ValueError as exc:
|
|
134
|
-
raise TunnelProtocolError("invalid tunnel frame status") from exc
|
|
135
|
-
try:
|
|
136
|
-
payload = base64.b64decode(payload_raw, validate=True)
|
|
137
|
-
except (binascii.Error, ValueError) as exc:
|
|
138
|
-
raise TunnelProtocolError("invalid tunnel frame base64 payload") from exc
|
|
139
|
-
frame = TunnelFrame(direction, sequence, kind, payload, status)
|
|
140
|
-
self._validate_frame(frame)
|
|
141
|
-
self._next_sequence += 1
|
|
142
|
-
return frame
|
|
143
16
|
|
|
144
|
-
@staticmethod
|
|
145
|
-
def _validate_frame(frame: TunnelFrame) -> None:
|
|
146
|
-
if not isinstance(frame.direction, Direction):
|
|
147
|
-
raise TunnelProtocolError("invalid tunnel frame direction")
|
|
148
|
-
if not isinstance(frame.sequence, int) or isinstance(frame.sequence, bool) or frame.sequence < 0:
|
|
149
|
-
raise TunnelProtocolError("invalid tunnel frame sequence")
|
|
150
|
-
if not isinstance(frame.kind, FrameKind):
|
|
151
|
-
raise TunnelProtocolError("invalid tunnel frame kind")
|
|
152
|
-
if not isinstance(frame.payload, bytes):
|
|
153
|
-
raise TunnelProtocolError("tunnel frame payload must be bytes")
|
|
154
|
-
if frame.kind is FrameKind.CLOSED:
|
|
155
|
-
if not isinstance(frame.status, int) or isinstance(frame.status, bool) or not 0 <= frame.status <= 255:
|
|
156
|
-
raise TunnelProtocolError("CLOSED tunnel frame requires an exit status")
|
|
157
|
-
elif frame.status is not None:
|
|
158
|
-
raise TunnelProtocolError("only CLOSED tunnel frames may contain a status")
|
|
159
|
-
if frame.kind not in {FrameKind.DATA, FrameKind.ERROR} and frame.payload:
|
|
160
|
-
raise TunnelProtocolError(f"{frame.kind.value} tunnel frame must not contain a payload")
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
class TunnelState(StrEnum):
|
|
164
|
-
CONNECTING = "CONNECTING"
|
|
165
|
-
OPEN = "OPEN"
|
|
166
|
-
LOCAL_EOF = "LOCAL_EOF"
|
|
167
|
-
REMOTE_EOF = "REMOTE_EOF"
|
|
168
|
-
DRAINING = "DRAINING"
|
|
169
|
-
ABORTING = "ABORTING"
|
|
170
|
-
CLOSED = "CLOSED"
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
class TunnelLifecycle:
|
|
174
|
-
def __init__(self) -> None:
|
|
175
|
-
self.state = TunnelState.CONNECTING
|
|
176
|
-
self.reusable = False
|
|
177
|
-
|
|
178
|
-
def mark_ready(self) -> None:
|
|
179
|
-
self._require(TunnelState.CONNECTING)
|
|
180
|
-
self.state = TunnelState.OPEN
|
|
181
|
-
|
|
182
|
-
def mark_local_eof(self) -> None:
|
|
183
|
-
if self.state is TunnelState.OPEN:
|
|
184
|
-
self.state = TunnelState.LOCAL_EOF
|
|
185
|
-
elif self.state is TunnelState.REMOTE_EOF:
|
|
186
|
-
self.state = TunnelState.DRAINING
|
|
187
|
-
else:
|
|
188
|
-
self._invalid("local EOF")
|
|
189
|
-
|
|
190
|
-
def mark_remote_eof(self) -> None:
|
|
191
|
-
if self.state is TunnelState.OPEN:
|
|
192
|
-
self.state = TunnelState.REMOTE_EOF
|
|
193
|
-
elif self.state is TunnelState.LOCAL_EOF:
|
|
194
|
-
self.state = TunnelState.DRAINING
|
|
195
|
-
else:
|
|
196
|
-
self._invalid("remote EOF")
|
|
197
|
-
|
|
198
|
-
def mark_closed(self, status: int) -> None:
|
|
199
|
-
self._require(TunnelState.DRAINING)
|
|
200
|
-
if not isinstance(status, int) or isinstance(status, bool) or not 0 <= status <= 255:
|
|
201
|
-
raise TunnelProtocolError("terminal tunnel status is invalid")
|
|
202
|
-
self.state = TunnelState.CLOSED
|
|
203
|
-
self.reusable = status == 0
|
|
204
17
|
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
self.state = TunnelState.ABORTING
|
|
208
|
-
self.reusable = False
|
|
18
|
+
class MuxTunnelStream(Protocol):
|
|
19
|
+
stream_id: int
|
|
209
20
|
|
|
210
|
-
def
|
|
211
|
-
self._require(TunnelState.ABORTING)
|
|
212
|
-
self.state = TunnelState.CLOSED
|
|
21
|
+
async def send(self, data: bytes) -> None: ...
|
|
213
22
|
|
|
214
|
-
def
|
|
215
|
-
if self.state is not expected:
|
|
216
|
-
raise TunnelProtocolError(f"tunnel state must be {expected.value}, found {self.state.value}")
|
|
23
|
+
async def send_eof(self) -> None: ...
|
|
217
24
|
|
|
218
|
-
def
|
|
219
|
-
raise TunnelProtocolError(f"cannot accept {event} while tunnel state is {self.state.value}")
|
|
25
|
+
async def receive(self, max_bytes: int = 64 * 1024) -> bytes | None: ...
|
|
220
26
|
|
|
27
|
+
async def finish(self) -> dict[str, object]: ...
|
|
221
28
|
|
|
222
|
-
|
|
223
|
-
def open(self) -> str: ...
|
|
29
|
+
async def reset(self, message: str = "stream reset") -> None: ...
|
|
224
30
|
|
|
225
|
-
def close(self, session_id: str, *, reusable: bool = True) -> None: ...
|
|
226
31
|
|
|
32
|
+
class MuxTunnelClient(Protocol):
|
|
33
|
+
@property
|
|
34
|
+
def closed(self) -> bool: ...
|
|
227
35
|
|
|
228
|
-
|
|
229
|
-
def __init__(self, pool: SessionPool, session_id: str) -> None:
|
|
230
|
-
self._pool = pool
|
|
231
|
-
self.session_id = session_id
|
|
232
|
-
self._close_task: asyncio.Task[None] | None = None
|
|
36
|
+
async def open_stream(self, method: str, params: dict[str, object]) -> MuxTunnelStream: ...
|
|
233
37
|
|
|
234
|
-
|
|
235
|
-
async def acquire(cls, pool: SessionPool) -> TunnelLease:
|
|
236
|
-
open_task = asyncio.create_task(asyncio.to_thread(pool.open))
|
|
237
|
-
try:
|
|
238
|
-
session_id = await asyncio.shield(open_task)
|
|
239
|
-
except asyncio.CancelledError:
|
|
240
|
-
await cls._wait_ignoring_cancellation(open_task)
|
|
241
|
-
if not open_task.cancelled() and open_task.exception() is None:
|
|
242
|
-
lease = cls(pool, open_task.result())
|
|
243
|
-
await lease.close(reusable=False)
|
|
244
|
-
raise
|
|
245
|
-
return cls(pool, session_id)
|
|
38
|
+
async def close(self) -> None: ...
|
|
246
39
|
|
|
247
|
-
async def close(self, *, reusable: bool) -> None:
|
|
248
|
-
if self._close_task is None:
|
|
249
|
-
self._close_task = asyncio.create_task(
|
|
250
|
-
asyncio.to_thread(self._pool.close, self.session_id, reusable=reusable)
|
|
251
|
-
)
|
|
252
|
-
await self._wait_ignoring_cancellation(self._close_task)
|
|
253
|
-
await self._close_task
|
|
254
40
|
|
|
255
|
-
|
|
256
|
-
async def _wait_ignoring_cancellation(task: asyncio.Task[object]) -> None:
|
|
257
|
-
while not task.done():
|
|
258
|
-
try:
|
|
259
|
-
await asyncio.shield(task)
|
|
260
|
-
except asyncio.CancelledError:
|
|
261
|
-
continue
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
class _TunnelFrameStream:
|
|
265
|
-
def __init__(self, client: HostBridgeClient, lease: TunnelLease, prefix: str) -> None:
|
|
266
|
-
self._client = client
|
|
267
|
-
self._lease = lease
|
|
268
|
-
self._codec = TunnelCodec(prefix, inbound_direction=Direction.DOWNSTREAM)
|
|
269
|
-
self._pending: deque[TunnelFrame] = deque()
|
|
270
|
-
|
|
271
|
-
async def next(self) -> TunnelFrame:
|
|
272
|
-
while not self._pending:
|
|
273
|
-
result = await asyncio.to_thread(
|
|
274
|
-
self._client.shell_read,
|
|
275
|
-
self._lease.session_id,
|
|
276
|
-
timeout=0.2,
|
|
277
|
-
max_bytes=TUNNEL_READ_BYTES,
|
|
278
|
-
owner="mock-ssh",
|
|
279
|
-
)
|
|
280
|
-
if result.data:
|
|
281
|
-
try:
|
|
282
|
-
self._pending.extend(self._codec.feed(result.data))
|
|
283
|
-
except TunnelProtocolError as exc:
|
|
284
|
-
raise HostBridgeError("protocol_mismatch", str(exc)) from exc
|
|
285
|
-
if not result.alive and not self._pending:
|
|
286
|
-
raise HostBridgeError(
|
|
287
|
-
"connection_lost",
|
|
288
|
-
"remote TCP tunnel closed without a terminal frame",
|
|
289
|
-
retryable=True,
|
|
290
|
-
)
|
|
291
|
-
return self._pending.popleft()
|
|
41
|
+
MuxClientFactory = Callable[[], Awaitable[MuxTunnelClient]]
|
|
292
42
|
|
|
293
43
|
|
|
294
44
|
class TunnelConnector:
|
|
295
45
|
def __init__(
|
|
296
46
|
self,
|
|
297
|
-
|
|
298
|
-
|
|
47
|
+
host_id: str,
|
|
48
|
+
client_factory: MuxClientFactory,
|
|
299
49
|
*,
|
|
300
|
-
connect_timeout:
|
|
50
|
+
connect_timeout: float,
|
|
301
51
|
) -> None:
|
|
52
|
+
if not isinstance(host_id, str) or not host_id:
|
|
53
|
+
raise ValueError("host_id must be a non-empty string")
|
|
302
54
|
if connect_timeout <= 0:
|
|
303
55
|
raise ValueError("connect_timeout must be positive")
|
|
304
|
-
self.
|
|
305
|
-
self.
|
|
306
|
-
self._connect_timeout = connect_timeout
|
|
56
|
+
self._host_id = host_id
|
|
57
|
+
self._client_factory = client_factory
|
|
58
|
+
self._connect_timeout = float(connect_timeout)
|
|
59
|
+
self._client: MuxTunnelClient | None = None
|
|
60
|
+
self._lock = asyncio.Lock()
|
|
61
|
+
self._closing = False
|
|
307
62
|
|
|
308
63
|
@staticmethod
|
|
309
64
|
def validate_destination(dest_host: str, dest_port: int) -> None:
|
|
@@ -312,109 +67,103 @@ class TunnelConnector:
|
|
|
312
67
|
if not isinstance(dest_port, int) or isinstance(dest_port, bool) or not 1 <= dest_port <= 65535:
|
|
313
68
|
raise ValueError("TCP forwarding destination port must be between 1 and 65535")
|
|
314
69
|
|
|
70
|
+
async def start(self) -> None:
|
|
71
|
+
await self._get_client()
|
|
72
|
+
|
|
315
73
|
async def connect(self, dest_host: str, dest_port: int) -> asyncssh.SSHTCPSession[bytes]:
|
|
316
74
|
self.validate_destination(dest_host, dest_port)
|
|
75
|
+
client: MuxTunnelClient | None = None
|
|
317
76
|
try:
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
try:
|
|
328
|
-
await asyncio.to_thread(
|
|
329
|
-
self._client.shell_write,
|
|
330
|
-
lease.session_id,
|
|
331
|
-
build_remote_tunnel_command(dest_host, dest_port, prefix),
|
|
332
|
-
owner="mock-ssh",
|
|
333
|
-
)
|
|
334
|
-
first = await asyncio.wait_for(frames.next(), timeout=self._connect_timeout)
|
|
335
|
-
if first.kind is FrameKind.ERROR:
|
|
336
|
-
raise HostBridgeError(
|
|
337
|
-
"connection_lost",
|
|
338
|
-
first.payload.decode("utf-8", errors="replace") or "remote TCP connection failed",
|
|
339
|
-
retryable=True,
|
|
340
|
-
)
|
|
341
|
-
if first.kind is not FrameKind.READY:
|
|
342
|
-
raise HostBridgeError(
|
|
343
|
-
"protocol_mismatch",
|
|
344
|
-
f"remote TCP tunnel returned {first.kind.value} before READY",
|
|
77
|
+
client = await self._get_client()
|
|
78
|
+
async with asyncio.timeout(self._connect_timeout):
|
|
79
|
+
stream = await client.open_stream(
|
|
80
|
+
"tcp.connect",
|
|
81
|
+
{
|
|
82
|
+
"host_id": self._host_id,
|
|
83
|
+
"destination_host": dest_host,
|
|
84
|
+
"destination_port": dest_port,
|
|
85
|
+
},
|
|
345
86
|
)
|
|
346
|
-
lifecycle.mark_ready()
|
|
347
87
|
except asyncio.CancelledError:
|
|
348
|
-
lifecycle.abort()
|
|
349
|
-
await lease.close(reusable=False)
|
|
350
|
-
lifecycle.finish_abort()
|
|
351
88
|
raise
|
|
352
|
-
except Exception as exc:
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
raise asyncssh.ChannelOpenError(asyncssh.OPEN_CONNECT_FAILED,
|
|
357
|
-
return DirectTCPIPSession(
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
89
|
+
except (Exception, TimeoutError) as exc:
|
|
90
|
+
if client is not None and client.closed:
|
|
91
|
+
await self._discard_client(client)
|
|
92
|
+
message = str(exc) or "TCP tunnel stream open timed out"
|
|
93
|
+
raise asyncssh.ChannelOpenError(asyncssh.OPEN_CONNECT_FAILED, message) from exc
|
|
94
|
+
return DirectTCPIPSession(stream, dest_host=dest_host, dest_port=dest_port)
|
|
95
|
+
|
|
96
|
+
async def close(self) -> None:
|
|
97
|
+
async with self._lock:
|
|
98
|
+
self._closing = True
|
|
99
|
+
client = self._client
|
|
100
|
+
if client is not None:
|
|
101
|
+
await client.close()
|
|
102
|
+
if self._client is client:
|
|
103
|
+
self._client = None
|
|
104
|
+
|
|
105
|
+
async def _get_client(self) -> MuxTunnelClient:
|
|
106
|
+
async with self._lock:
|
|
107
|
+
if self._closing:
|
|
108
|
+
raise RuntimeError("TCP tunnel connector is closed")
|
|
109
|
+
if self._client is not None and not self._client.closed:
|
|
110
|
+
return self._client
|
|
111
|
+
stale = self._client
|
|
112
|
+
async with asyncio.timeout(self._connect_timeout):
|
|
113
|
+
client = await self._client_factory()
|
|
114
|
+
self._client = client
|
|
115
|
+
if stale is not None and stale is not client:
|
|
116
|
+
with suppress(Exception):
|
|
117
|
+
await stale.close()
|
|
118
|
+
return client
|
|
119
|
+
|
|
120
|
+
async def _discard_client(self, client: MuxTunnelClient) -> None:
|
|
121
|
+
async with self._lock:
|
|
122
|
+
if self._client is client:
|
|
123
|
+
self._client = None
|
|
366
124
|
|
|
367
125
|
|
|
368
126
|
_INPUT_EOF = object()
|
|
369
127
|
|
|
370
128
|
|
|
371
129
|
class DirectTCPIPSession(asyncssh.SSHTCPSession[bytes]):
|
|
372
|
-
def __init__(
|
|
373
|
-
self
|
|
374
|
-
client: HostBridgeClient,
|
|
375
|
-
lease: TunnelLease,
|
|
376
|
-
prefix: str,
|
|
377
|
-
frames: _TunnelFrameStream,
|
|
378
|
-
lifecycle: TunnelLifecycle,
|
|
379
|
-
*,
|
|
380
|
-
dest_host: str,
|
|
381
|
-
dest_port: int,
|
|
382
|
-
) -> None:
|
|
383
|
-
self._client = client
|
|
384
|
-
self._lease = lease
|
|
385
|
-
self._codec = TunnelCodec(prefix, inbound_direction=Direction.DOWNSTREAM)
|
|
386
|
-
self._frames = frames
|
|
387
|
-
self._lifecycle = lifecycle
|
|
130
|
+
def __init__(self, stream: MuxTunnelStream, *, dest_host: str, dest_port: int) -> None:
|
|
131
|
+
self._stream = stream
|
|
388
132
|
self._dest_host = dest_host
|
|
389
133
|
self._dest_port = dest_port
|
|
390
134
|
self._channel: asyncssh.SSHTCPChannel[bytes] | None = None
|
|
391
|
-
self._input:
|
|
135
|
+
self._input: deque[bytes | object] = deque()
|
|
136
|
+
self._input_ready = asyncio.Event()
|
|
392
137
|
self._input_buffered_bytes = 0
|
|
393
138
|
self._input_paused = False
|
|
394
|
-
self.
|
|
139
|
+
self._input_eof = False
|
|
395
140
|
self._write_ready = asyncio.Event()
|
|
396
141
|
self._write_ready.set()
|
|
397
142
|
self._runner: asyncio.Task[None] | None = None
|
|
398
|
-
self.
|
|
143
|
+
self._cleanup_task: asyncio.Task[None] | None = None
|
|
399
144
|
self._finished = asyncio.Event()
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
def state(self) -> TunnelState:
|
|
403
|
-
return self._lifecycle.state
|
|
145
|
+
self._aborted = False
|
|
146
|
+
self._reset_sent = False
|
|
404
147
|
|
|
405
148
|
def connection_made(self, chan: asyncssh.SSHTCPChannel[bytes]) -> None:
|
|
406
149
|
self._channel = chan
|
|
407
150
|
|
|
408
151
|
def session_started(self) -> None:
|
|
409
|
-
self._runner
|
|
152
|
+
if self._runner is not None or self._finished.is_set():
|
|
153
|
+
return
|
|
154
|
+
self._runner = asyncio.create_task(
|
|
155
|
+
self._run(),
|
|
156
|
+
name=f"hostbridge-mock-ssh-tcp-{self._stream.stream_id}",
|
|
157
|
+
)
|
|
410
158
|
self._runner.add_done_callback(self._runner_done)
|
|
411
159
|
|
|
412
160
|
def data_received(self, data: bytes, datatype: asyncssh.DataType) -> None: # noqa: ARG002
|
|
413
|
-
if not data:
|
|
161
|
+
if not data or self._input_eof or self._finished.is_set():
|
|
414
162
|
return
|
|
415
163
|
payload = bytes(data)
|
|
416
|
-
self._input.
|
|
164
|
+
self._input.append(payload)
|
|
417
165
|
self._input_buffered_bytes += len(payload)
|
|
166
|
+
self._input_ready.set()
|
|
418
167
|
if (
|
|
419
168
|
self._input_buffered_bytes >= TUNNEL_HIGH_WATER_BYTES
|
|
420
169
|
and not self._input_paused
|
|
@@ -424,7 +173,10 @@ class DirectTCPIPSession(asyncssh.SSHTCPSession[bytes]):
|
|
|
424
173
|
self._input_paused = True
|
|
425
174
|
|
|
426
175
|
def eof_received(self) -> bool:
|
|
427
|
-
self.
|
|
176
|
+
if not self._input_eof and not self._finished.is_set():
|
|
177
|
+
self._input_eof = True
|
|
178
|
+
self._input.append(_INPUT_EOF)
|
|
179
|
+
self._input_ready.set()
|
|
428
180
|
return True
|
|
429
181
|
|
|
430
182
|
def connection_lost(self, exc: Exception | None) -> None: # noqa: ARG002
|
|
@@ -437,41 +189,36 @@ class DirectTCPIPSession(asyncssh.SSHTCPSession[bytes]):
|
|
|
437
189
|
self._write_ready.set()
|
|
438
190
|
|
|
439
191
|
def abort(self) -> None:
|
|
440
|
-
self.
|
|
192
|
+
if self._finished.is_set():
|
|
193
|
+
return
|
|
194
|
+
self._aborted = True
|
|
441
195
|
if self._runner is not None and self._runner is not asyncio.current_task() and not self._runner.done():
|
|
442
196
|
self._runner.cancel()
|
|
443
|
-
elif self._runner is None:
|
|
444
|
-
self.
|
|
197
|
+
elif self._runner is None and self._cleanup_task is None:
|
|
198
|
+
self._cleanup_task = asyncio.create_task(self._finish_orphaned_session())
|
|
445
199
|
|
|
446
200
|
async def wait_finished(self) -> None:
|
|
447
201
|
await self._finished.wait()
|
|
448
202
|
|
|
449
203
|
def _runner_done(self, task: asyncio.Task[None]) -> None: # noqa: ARG002
|
|
450
|
-
if not self._finished.is_set():
|
|
451
|
-
self.
|
|
452
|
-
|
|
453
|
-
def _schedule_orphan_cleanup(self) -> None:
|
|
454
|
-
if self._orphan_cleanup is None:
|
|
455
|
-
self._orphan_cleanup = asyncio.create_task(self._finish_orphaned_session())
|
|
204
|
+
if not self._finished.is_set() and self._cleanup_task is None:
|
|
205
|
+
self._cleanup_task = asyncio.create_task(self._finish_orphaned_session())
|
|
456
206
|
|
|
457
207
|
async def _finish_orphaned_session(self) -> None:
|
|
458
|
-
self.
|
|
459
|
-
|
|
460
|
-
if self._lifecycle.state is TunnelState.ABORTING:
|
|
461
|
-
self._lifecycle.finish_abort()
|
|
462
|
-
self._close_channel()
|
|
463
|
-
self._finished.set()
|
|
208
|
+
await self._reset_stream()
|
|
209
|
+
self._finish_channel()
|
|
464
210
|
|
|
465
211
|
async def _run(self) -> None:
|
|
466
212
|
input_task = asyncio.create_task(self._pump_input())
|
|
467
213
|
output_task = asyncio.create_task(self._pump_output())
|
|
214
|
+
clean = False
|
|
468
215
|
try:
|
|
469
216
|
await asyncio.gather(input_task, output_task)
|
|
217
|
+
await self._stream.finish()
|
|
218
|
+
clean = True
|
|
470
219
|
except asyncio.CancelledError:
|
|
471
|
-
self._lifecycle.abort()
|
|
472
220
|
raise
|
|
473
221
|
except Exception as exc:
|
|
474
|
-
self._lifecycle.abort()
|
|
475
222
|
print(
|
|
476
223
|
f"hostbridge mock-ssh TCP forwarding to {self._dest_host}:{self._dest_port} closed: {exc}",
|
|
477
224
|
file=sys.stderr,
|
|
@@ -482,111 +229,61 @@ class DirectTCPIPSession(asyncssh.SSHTCPSession[bytes]):
|
|
|
482
229
|
if not task.done():
|
|
483
230
|
task.cancel()
|
|
484
231
|
await asyncio.gather(input_task, output_task, return_exceptions=True)
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
self._close_channel()
|
|
489
|
-
self._finished.set()
|
|
232
|
+
if not clean:
|
|
233
|
+
await self._reset_stream()
|
|
234
|
+
self._finish_channel()
|
|
490
235
|
|
|
491
236
|
async def _pump_input(self) -> None:
|
|
492
237
|
while True:
|
|
493
|
-
item = await self.
|
|
238
|
+
item = await self._next_input()
|
|
494
239
|
if item is _INPUT_EOF:
|
|
495
|
-
await self.
|
|
240
|
+
await self._stream.send_eof()
|
|
496
241
|
return
|
|
497
|
-
|
|
498
242
|
if not isinstance(item, bytes):
|
|
499
|
-
raise
|
|
500
|
-
batch = bytearray(item)
|
|
501
|
-
saw_eof = False
|
|
502
|
-
while not self._input.empty():
|
|
503
|
-
queued = self._input.get_nowait()
|
|
504
|
-
if queued is _INPUT_EOF:
|
|
505
|
-
saw_eof = True
|
|
506
|
-
break
|
|
507
|
-
if not isinstance(queued, bytes):
|
|
508
|
-
raise RuntimeError("TCP tunnel input queue contained an invalid item")
|
|
509
|
-
batch.extend(queued)
|
|
243
|
+
raise TypeError("tunnel input queue contains a non-bytes item")
|
|
510
244
|
try:
|
|
511
|
-
|
|
512
|
-
await self._send_data(bytes(batch[offset : offset + TUNNEL_WRITE_BATCH_BYTES]))
|
|
245
|
+
await self._stream.send(item)
|
|
513
246
|
finally:
|
|
514
|
-
self._input_buffered_bytes -= len(
|
|
247
|
+
self._input_buffered_bytes -= len(item)
|
|
515
248
|
self._resume_input_if_ready()
|
|
516
|
-
if saw_eof:
|
|
517
|
-
await self._send_eof()
|
|
518
|
-
return
|
|
519
249
|
|
|
520
|
-
async def
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
)
|
|
530
|
-
)
|
|
531
|
-
self._upstream_sequence += 1
|
|
532
|
-
await self._write_frames(frames)
|
|
533
|
-
|
|
534
|
-
async def _send_eof(self) -> None:
|
|
535
|
-
frame = TunnelFrame(Direction.UPSTREAM, self._upstream_sequence, FrameKind.EOF)
|
|
536
|
-
self._upstream_sequence += 1
|
|
537
|
-
await self._write_frames([frame])
|
|
538
|
-
self._lifecycle.mark_local_eof()
|
|
539
|
-
|
|
540
|
-
async def _write_frames(self, frames: list[TunnelFrame]) -> None:
|
|
541
|
-
payload = b"".join(self._codec.encode(frame) for frame in frames)
|
|
542
|
-
await asyncio.to_thread(
|
|
543
|
-
self._client.shell_write,
|
|
544
|
-
self._lease.session_id,
|
|
545
|
-
payload,
|
|
546
|
-
owner="mock-ssh",
|
|
547
|
-
)
|
|
250
|
+
async def _next_input(self) -> bytes | object:
|
|
251
|
+
while not self._input:
|
|
252
|
+
self._input_ready.clear()
|
|
253
|
+
if not self._input:
|
|
254
|
+
await self._input_ready.wait()
|
|
255
|
+
item = self._input.popleft()
|
|
256
|
+
if not self._input:
|
|
257
|
+
self._input_ready.clear()
|
|
258
|
+
return item
|
|
548
259
|
|
|
549
260
|
async def _pump_output(self) -> None:
|
|
550
261
|
if self._channel is None:
|
|
551
262
|
raise RuntimeError("TCP tunnel channel was not initialized")
|
|
552
263
|
while True:
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
raise HostBridgeError("protocol_mismatch", "remote TCP data arrived after EOF")
|
|
557
|
-
await self._write_ready.wait()
|
|
558
|
-
self._channel.write(frame.payload)
|
|
559
|
-
elif frame.kind is FrameKind.EOF:
|
|
560
|
-
self._lifecycle.mark_remote_eof()
|
|
264
|
+
await self._write_ready.wait()
|
|
265
|
+
chunk = await self._stream.receive()
|
|
266
|
+
if chunk is None:
|
|
561
267
|
self._channel.write_eof()
|
|
562
|
-
elif frame.kind is FrameKind.ERROR:
|
|
563
|
-
raise HostBridgeError(
|
|
564
|
-
"connection_lost",
|
|
565
|
-
frame.payload.decode("utf-8", errors="replace") or "remote TCP tunnel failed",
|
|
566
|
-
retryable=True,
|
|
567
|
-
)
|
|
568
|
-
elif frame.kind is FrameKind.CLOSED:
|
|
569
|
-
assert frame.status is not None
|
|
570
|
-
self._lifecycle.mark_closed(frame.status)
|
|
571
|
-
if frame.status != 0:
|
|
572
|
-
raise HostBridgeError(
|
|
573
|
-
"connection_lost",
|
|
574
|
-
f"remote TCP tunnel exited with status {frame.status}",
|
|
575
|
-
retryable=True,
|
|
576
|
-
)
|
|
577
268
|
return
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
269
|
+
self._channel.write(chunk)
|
|
270
|
+
|
|
271
|
+
async def _reset_stream(self) -> None:
|
|
272
|
+
if self._reset_sent:
|
|
273
|
+
return
|
|
274
|
+
self._reset_sent = True
|
|
275
|
+
with suppress(Exception):
|
|
276
|
+
await self._stream.reset("mock-ssh TCP channel closed")
|
|
583
277
|
|
|
584
278
|
def _resume_input_if_ready(self) -> None:
|
|
585
279
|
if self._input_paused and self._input_buffered_bytes <= TUNNEL_LOW_WATER_BYTES and self._channel is not None:
|
|
586
280
|
self._channel.resume_reading()
|
|
587
281
|
self._input_paused = False
|
|
588
282
|
|
|
589
|
-
def
|
|
283
|
+
def _finish_channel(self) -> None:
|
|
284
|
+
if self._finished.is_set():
|
|
285
|
+
return
|
|
286
|
+
self._finished.set()
|
|
590
287
|
if self._channel is not None:
|
|
591
288
|
with suppress(Exception):
|
|
592
289
|
self._channel.close()
|