@neoline/hostbridge 2.0.4 → 2.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -9
- package/docs/ARCHITECTURE.md +62 -0
- package/package.json +2 -1
- package/pyproject.toml +1 -1
- package/src/server_control_mcp/__init__.py +4 -3
- package/src/server_control_mcp/async_lifecycle.py +41 -0
- package/src/server_control_mcp/cli.py +1 -1
- package/src/server_control_mcp/client.py +302 -287
- package/src/server_control_mcp/config.py +2 -1
- package/src/server_control_mcp/daemon.py +233 -526
- package/src/server_control_mcp/doctor.py +34 -2
- package/src/server_control_mcp/hosts.py +136 -2
- package/src/server_control_mcp/mock_ssh.py +52 -14
- package/src/server_control_mcp/mock_ssh_exec.py +147 -58
- package/src/server_control_mcp/mock_ssh_session.py +3 -2
- package/src/server_control_mcp/mock_ssh_sftp.py +126 -28
- package/src/server_control_mcp/mock_ssh_tunnel.py +141 -444
- package/src/server_control_mcp/mux_connection.py +326 -0
- package/src/server_control_mcp/mux_daemon.py +186 -0
- package/src/server_control_mcp/mux_protocol.py +279 -0
- package/src/server_control_mcp/mux_records.py +71 -0
- package/src/server_control_mcp/mux_rpc.py +1779 -0
- package/src/server_control_mcp/mux_service.py +506 -0
- package/src/server_control_mcp/mux_stream.py +283 -0
- package/src/server_control_mcp/mux_sync_client.py +224 -0
- package/src/server_control_mcp/remote_agent_bundle.py +56 -0
- package/src/server_control_mcp/remote_mux_agent.py +769 -0
- package/src/server_control_mcp/runtime_services.py +862 -0
- package/src/server_control_mcp/server.py +33 -18
- package/src/server_control_mcp/services.py +2 -666
- package/src/server_control_mcp/transports/__init__.py +4 -8
- package/src/server_control_mcp/transports/base.py +60 -38
- package/src/server_control_mcp/transports/ssh.py +206 -259
- package/src/server_control_mcp/tunnel_manager.py +1245 -0
- package/src/server_control_mcp/tunnel_native_ssh.py +454 -0
- package/src/server_control_mcp/tunnel_providers.py +20 -0
- package/src/server_control_mcp/tunnel_pty_agent.py +909 -0
- package/src/server_control_mcp/protocol.py +0 -363
- package/src/server_control_mcp/remote_tunnel_agent.py +0 -143
- package/src/server_control_mcp/transports/pty.py +0 -515
|
@@ -10,6 +10,7 @@ from typing import Protocol
|
|
|
10
10
|
|
|
11
11
|
from . import __version__
|
|
12
12
|
from .config import load_v1_config
|
|
13
|
+
from .mux_protocol import MUX_PROTOCOL_VERSION
|
|
13
14
|
from .runtime import runtime_paths
|
|
14
15
|
|
|
15
16
|
|
|
@@ -17,6 +18,9 @@ class DoctorClient(Protocol):
|
|
|
17
18
|
def hello(self) -> dict[str, object]:
|
|
18
19
|
raise NotImplementedError
|
|
19
20
|
|
|
21
|
+
def tunnel_health(self) -> dict[str, object]:
|
|
22
|
+
raise NotImplementedError
|
|
23
|
+
|
|
20
24
|
|
|
21
25
|
@dataclass(frozen=True, slots=True)
|
|
22
26
|
class DiagnosticResult:
|
|
@@ -46,6 +50,7 @@ def collect_diagnostics(
|
|
|
46
50
|
_config_diagnostic(config_path),
|
|
47
51
|
_runtime_diagnostic(),
|
|
48
52
|
_daemon_diagnostic(client),
|
|
53
|
+
_tunnel_diagnostic(client),
|
|
49
54
|
_executable_diagnostic(executable_paths),
|
|
50
55
|
]
|
|
51
56
|
return DoctorReport(tuple(results))
|
|
@@ -97,15 +102,42 @@ def _daemon_diagnostic(client: DoctorClient) -> DiagnosticResult:
|
|
|
97
102
|
hello = client.hello()
|
|
98
103
|
except Exception as exc:
|
|
99
104
|
return DiagnosticResult("daemon", "fail", f"daemon unavailable: {exc}", {})
|
|
100
|
-
valid =
|
|
105
|
+
valid = (
|
|
106
|
+
hello.get("service") == "hostbridge"
|
|
107
|
+
and hello.get("protocol") == MUX_PROTOCOL_VERSION
|
|
108
|
+
and hello.get("version") == __version__
|
|
109
|
+
)
|
|
101
110
|
return DiagnosticResult(
|
|
102
111
|
"daemon",
|
|
103
112
|
"pass" if valid else "fail",
|
|
104
|
-
"daemon protocol
|
|
113
|
+
f"daemon protocol v{MUX_PROTOCOL_VERSION} is healthy" if valid else "daemon returned an incompatible handshake",
|
|
105
114
|
dict(hello),
|
|
106
115
|
)
|
|
107
116
|
|
|
108
117
|
|
|
118
|
+
def _tunnel_diagnostic(client: DoctorClient) -> DiagnosticResult:
|
|
119
|
+
try:
|
|
120
|
+
result = client.tunnel_health()
|
|
121
|
+
except Exception as exc:
|
|
122
|
+
return DiagnosticResult("tunnel", "fail", f"tunnel diagnostics unavailable: {exc}", {})
|
|
123
|
+
hosts = result.get("hosts")
|
|
124
|
+
if not isinstance(hosts, list) or any(not isinstance(host, dict) for host in hosts):
|
|
125
|
+
return DiagnosticResult("tunnel", "fail", "daemon returned invalid tunnel diagnostics", dict(result))
|
|
126
|
+
enabled = [host for host in hosts if host.get("enabled") is True]
|
|
127
|
+
failed = [host for host in enabled if host.get("state") == "failed"]
|
|
128
|
+
inactive = [host for host in enabled if host.get("state") != "ready"]
|
|
129
|
+
if failed:
|
|
130
|
+
status = "fail"
|
|
131
|
+
message = f"{len(failed)} tunnel provider runtime(s) failed"
|
|
132
|
+
elif inactive:
|
|
133
|
+
status = "warn"
|
|
134
|
+
message = f"{len(inactive)} tunnel provider runtime(s) configured but not handshaken"
|
|
135
|
+
else:
|
|
136
|
+
status = "pass"
|
|
137
|
+
message = "tunnel provider runtimes are ready"
|
|
138
|
+
return DiagnosticResult("tunnel", status, message, dict(result))
|
|
139
|
+
|
|
140
|
+
|
|
109
141
|
def _executable_diagnostic(paths: list[str]) -> DiagnosticResult:
|
|
110
142
|
if not paths:
|
|
111
143
|
return DiagnosticResult("executable", "fail", "hostbridge executable was not found", {"paths": []})
|
|
@@ -4,6 +4,7 @@ import os
|
|
|
4
4
|
import re
|
|
5
5
|
from collections.abc import Iterable, Sequence
|
|
6
6
|
from dataclasses import dataclass, field
|
|
7
|
+
from ipaddress import ip_address, ip_network
|
|
7
8
|
from pathlib import Path
|
|
8
9
|
|
|
9
10
|
from .config import load_v1_config
|
|
@@ -42,6 +43,17 @@ class SshHostConfig:
|
|
|
42
43
|
extra_args: tuple[str, ...] = ()
|
|
43
44
|
|
|
44
45
|
|
|
46
|
+
@dataclass(frozen=True, slots=True)
|
|
47
|
+
class ForwardingPolicy:
|
|
48
|
+
enabled: bool = False
|
|
49
|
+
allow_cidrs: tuple[str, ...] = ()
|
|
50
|
+
allow_hostnames: tuple[str, ...] = ()
|
|
51
|
+
allow_ports: tuple[int, ...] = ()
|
|
52
|
+
max_streams: int = 128
|
|
53
|
+
idle_timeout_seconds: float = 300
|
|
54
|
+
stream_deadline_seconds: float = 24 * 60 * 60
|
|
55
|
+
|
|
56
|
+
|
|
45
57
|
@dataclass(frozen=True, slots=True)
|
|
46
58
|
class HostConfig:
|
|
47
59
|
"""A single approved connection target."""
|
|
@@ -58,6 +70,7 @@ class HostConfig:
|
|
|
58
70
|
transport: str = "auto"
|
|
59
71
|
limits: HostLimits = field(default_factory=HostLimits)
|
|
60
72
|
ssh: SshHostConfig | None = None
|
|
73
|
+
forwarding: ForwardingPolicy = field(default_factory=ForwardingPolicy)
|
|
61
74
|
|
|
62
75
|
@classmethod
|
|
63
76
|
def script(
|
|
@@ -68,10 +81,19 @@ class HostConfig:
|
|
|
68
81
|
*,
|
|
69
82
|
transport: str = "auto",
|
|
70
83
|
limits: HostLimits | None = None,
|
|
84
|
+
forwarding: ForwardingPolicy | None = None,
|
|
71
85
|
) -> HostConfig:
|
|
72
86
|
path = Path(script).expanduser()
|
|
73
87
|
return cls(
|
|
74
|
-
host_id,
|
|
88
|
+
host_id,
|
|
89
|
+
label,
|
|
90
|
+
"/bin/sh",
|
|
91
|
+
[str(path)],
|
|
92
|
+
"script",
|
|
93
|
+
path,
|
|
94
|
+
transport=transport,
|
|
95
|
+
limits=limits or HostLimits(),
|
|
96
|
+
forwarding=forwarding or ForwardingPolicy(),
|
|
75
97
|
)
|
|
76
98
|
|
|
77
99
|
@classmethod
|
|
@@ -89,6 +111,7 @@ class HostConfig:
|
|
|
89
111
|
transport: str = "auto",
|
|
90
112
|
limits: HostLimits | None = None,
|
|
91
113
|
ssh: SshHostConfig | None = None,
|
|
114
|
+
forwarding: ForwardingPolicy | None = None,
|
|
92
115
|
) -> HostConfig:
|
|
93
116
|
return cls(
|
|
94
117
|
host_id,
|
|
@@ -103,6 +126,7 @@ class HostConfig:
|
|
|
103
126
|
transport,
|
|
104
127
|
limits or HostLimits(),
|
|
105
128
|
ssh,
|
|
129
|
+
forwarding or ForwardingPolicy(),
|
|
106
130
|
)
|
|
107
131
|
|
|
108
132
|
def describe(self) -> dict[str, object]:
|
|
@@ -113,6 +137,12 @@ class HostConfig:
|
|
|
113
137
|
"command": self.command,
|
|
114
138
|
"args": list(self.args),
|
|
115
139
|
"transport": self.transport,
|
|
140
|
+
"forwarding": {
|
|
141
|
+
"enabled": self.forwarding.enabled,
|
|
142
|
+
"max_streams": self.forwarding.max_streams,
|
|
143
|
+
"idle_timeout_seconds": self.forwarding.idle_timeout_seconds,
|
|
144
|
+
"stream_deadline_seconds": self.forwarding.stream_deadline_seconds,
|
|
145
|
+
},
|
|
116
146
|
}
|
|
117
147
|
if self.script_path is not None:
|
|
118
148
|
info.update(
|
|
@@ -181,12 +211,20 @@ def _parse_host(item: object, path: Path, index: int) -> HostConfig:
|
|
|
181
211
|
if transport == "ssh" and "ssh" not in item:
|
|
182
212
|
raise ValueError(f"{path} hosts[{index}].transport ssh requires an ssh connection")
|
|
183
213
|
limits = _parse_limits(item.get("limits"), path, index)
|
|
214
|
+
forwarding = _parse_forwarding(item.get("forwarding"), path, index)
|
|
184
215
|
|
|
185
216
|
if "script" in item:
|
|
186
217
|
script = str(item.get("script", "")).strip()
|
|
187
218
|
if not script:
|
|
188
219
|
raise ValueError(f"{path} hosts[{index}].script must not be empty")
|
|
189
|
-
host = HostConfig.script(
|
|
220
|
+
host = HostConfig.script(
|
|
221
|
+
host_id,
|
|
222
|
+
label,
|
|
223
|
+
script,
|
|
224
|
+
transport=transport,
|
|
225
|
+
limits=limits,
|
|
226
|
+
forwarding=forwarding,
|
|
227
|
+
)
|
|
190
228
|
return HostConfig(
|
|
191
229
|
host.id,
|
|
192
230
|
host.label,
|
|
@@ -199,6 +237,7 @@ def _parse_host(item: object, path: Path, index: int) -> HostConfig:
|
|
|
199
237
|
ready_expect,
|
|
200
238
|
transport,
|
|
201
239
|
limits,
|
|
240
|
+
forwarding=forwarding,
|
|
202
241
|
)
|
|
203
242
|
|
|
204
243
|
if "ssh" in item:
|
|
@@ -219,6 +258,7 @@ def _parse_host(item: object, path: Path, index: int) -> HostConfig:
|
|
|
219
258
|
transport=transport,
|
|
220
259
|
limits=limits,
|
|
221
260
|
ssh=ssh_config,
|
|
261
|
+
forwarding=forwarding,
|
|
222
262
|
)
|
|
223
263
|
|
|
224
264
|
command = str(item.get("command", "")).strip()
|
|
@@ -237,6 +277,100 @@ def _parse_host(item: object, path: Path, index: int) -> HostConfig:
|
|
|
237
277
|
ready_expect=ready_expect,
|
|
238
278
|
transport=transport,
|
|
239
279
|
limits=limits,
|
|
280
|
+
forwarding=forwarding,
|
|
281
|
+
)
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def _parse_forwarding(raw: object, path: Path, index: int) -> ForwardingPolicy:
|
|
285
|
+
if raw is None:
|
|
286
|
+
return ForwardingPolicy()
|
|
287
|
+
if not isinstance(raw, dict):
|
|
288
|
+
raise ValueError(f"{path} hosts[{index}].forwarding must be an object")
|
|
289
|
+
allowed_fields = {
|
|
290
|
+
"enabled",
|
|
291
|
+
"allow_cidrs",
|
|
292
|
+
"allow_hostnames",
|
|
293
|
+
"allow_ports",
|
|
294
|
+
"max_streams",
|
|
295
|
+
"idle_timeout_seconds",
|
|
296
|
+
"stream_deadline_seconds",
|
|
297
|
+
}
|
|
298
|
+
unknown = sorted(set(raw) - allowed_fields)
|
|
299
|
+
if unknown:
|
|
300
|
+
raise ValueError(f"{path} hosts[{index}].forwarding contains unknown field {unknown[0]!r}")
|
|
301
|
+
|
|
302
|
+
enabled = raw.get("enabled", False)
|
|
303
|
+
if not isinstance(enabled, bool):
|
|
304
|
+
raise ValueError(f"{path} hosts[{index}].forwarding.enabled must be a boolean")
|
|
305
|
+
|
|
306
|
+
raw_cidrs = raw.get("allow_cidrs", [])
|
|
307
|
+
if not isinstance(raw_cidrs, list) or not all(isinstance(value, str) for value in raw_cidrs):
|
|
308
|
+
raise ValueError(f"{path} hosts[{index}].forwarding.allow_cidrs must be a list of CIDR strings")
|
|
309
|
+
cidrs: list[str] = []
|
|
310
|
+
for value in raw_cidrs:
|
|
311
|
+
try:
|
|
312
|
+
network = ip_network(value, strict=True)
|
|
313
|
+
except ValueError as exc:
|
|
314
|
+
raise ValueError(f"{path} hosts[{index}].forwarding contains invalid CIDR {value!r}") from exc
|
|
315
|
+
cidrs.append(str(network))
|
|
316
|
+
|
|
317
|
+
raw_hostnames = raw.get("allow_hostnames", [])
|
|
318
|
+
if not isinstance(raw_hostnames, list) or not all(isinstance(value, str) for value in raw_hostnames):
|
|
319
|
+
raise ValueError(f"{path} hosts[{index}].forwarding.allow_hostnames must be a list of hostnames")
|
|
320
|
+
hostnames: list[str] = []
|
|
321
|
+
for value in raw_hostnames:
|
|
322
|
+
hostname = value.strip().rstrip(".").lower()
|
|
323
|
+
if not _valid_exact_hostname(hostname):
|
|
324
|
+
raise ValueError(f"{path} hosts[{index}].forwarding contains invalid hostname {value!r}")
|
|
325
|
+
hostnames.append(hostname)
|
|
326
|
+
|
|
327
|
+
raw_ports = raw.get("allow_ports", [])
|
|
328
|
+
if not isinstance(raw_ports, list) or any(
|
|
329
|
+
not isinstance(port, int) or isinstance(port, bool) or not 1 <= port <= 65535 for port in raw_ports
|
|
330
|
+
):
|
|
331
|
+
raise ValueError(f"{path} hosts[{index}].forwarding.allow_ports must contain valid TCP ports")
|
|
332
|
+
|
|
333
|
+
max_streams = raw.get("max_streams", 128)
|
|
334
|
+
if not isinstance(max_streams, int) or isinstance(max_streams, bool) or not 1 <= max_streams <= 4096:
|
|
335
|
+
raise ValueError(f"{path} hosts[{index}].forwarding.max_streams must be between 1 and 4096")
|
|
336
|
+
idle_timeout = raw.get("idle_timeout_seconds", 300)
|
|
337
|
+
if not isinstance(idle_timeout, (int, float)) or isinstance(idle_timeout, bool) or not 0 < idle_timeout <= 86400:
|
|
338
|
+
raise ValueError(f"{path} hosts[{index}].forwarding.idle_timeout_seconds must be between 0 and 86400")
|
|
339
|
+
stream_deadline = raw.get("stream_deadline_seconds", 24 * 60 * 60)
|
|
340
|
+
if (
|
|
341
|
+
not isinstance(stream_deadline, (int, float))
|
|
342
|
+
or isinstance(stream_deadline, bool)
|
|
343
|
+
or not 0 < stream_deadline <= 7 * 24 * 60 * 60
|
|
344
|
+
):
|
|
345
|
+
raise ValueError(f"{path} hosts[{index}].forwarding.stream_deadline_seconds must be between 0 and 604800")
|
|
346
|
+
|
|
347
|
+
return ForwardingPolicy(
|
|
348
|
+
enabled=enabled,
|
|
349
|
+
allow_cidrs=tuple(dict.fromkeys(cidrs)),
|
|
350
|
+
allow_hostnames=tuple(dict.fromkeys(hostnames)),
|
|
351
|
+
allow_ports=tuple(dict.fromkeys(raw_ports)),
|
|
352
|
+
max_streams=max_streams,
|
|
353
|
+
idle_timeout_seconds=float(idle_timeout),
|
|
354
|
+
stream_deadline_seconds=float(stream_deadline),
|
|
355
|
+
)
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _valid_exact_hostname(hostname: str) -> bool:
|
|
359
|
+
if not hostname or len(hostname) > 253 or "*" in hostname or "\x00" in hostname:
|
|
360
|
+
return False
|
|
361
|
+
try:
|
|
362
|
+
ip_address(hostname)
|
|
363
|
+
except ValueError:
|
|
364
|
+
pass
|
|
365
|
+
else:
|
|
366
|
+
return False
|
|
367
|
+
labels = hostname.split(".")
|
|
368
|
+
return all(
|
|
369
|
+
1 <= len(label) <= 63
|
|
370
|
+
and label[0].isalnum()
|
|
371
|
+
and label[-1].isalnum()
|
|
372
|
+
and all(character.isalnum() or character == "-" for character in label)
|
|
373
|
+
for label in labels
|
|
240
374
|
)
|
|
241
375
|
|
|
242
376
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import asyncio
|
|
4
|
+
import sys
|
|
4
5
|
from collections.abc import Awaitable
|
|
5
6
|
from pathlib import Path
|
|
6
7
|
from typing import Any, cast
|
|
@@ -12,11 +13,14 @@ from .mock_ssh_exec import HostBridgeSSHProcess
|
|
|
12
13
|
from .mock_ssh_session import HostBridgeSessionFactory
|
|
13
14
|
from .mock_ssh_sftp import HostBridgeSFTPServer
|
|
14
15
|
from .mock_ssh_tunnel import TunnelConnector
|
|
16
|
+
from .mux_rpc import MuxRpcClient
|
|
15
17
|
from .secure_files import create_private_file, ensure_private_directory, replace_private_file, validate_private_file
|
|
16
18
|
|
|
17
19
|
DEFAULT_PORT_RANGE = range(2222, 2300)
|
|
18
20
|
DEFAULT_COMMAND_TIMEOUT = 60
|
|
19
21
|
DEFAULT_MAX_TRANSFER_BYTES = 1024 * 1024 * 1024
|
|
22
|
+
DEFAULT_SFTP_MAX_OPEN_HANDLES = 16
|
|
23
|
+
DEFAULT_SFTP_MAX_TEMP_BYTES = 1024 * 1024 * 1024
|
|
20
24
|
|
|
21
25
|
|
|
22
26
|
class HostBridgeSSHServer(asyncssh.SSHServer):
|
|
@@ -225,7 +229,7 @@ async def serve_mock_ssh(
|
|
|
225
229
|
if config_path is not None:
|
|
226
230
|
raise HostBridgeError(
|
|
227
231
|
"invalid_request",
|
|
228
|
-
"mock-ssh
|
|
232
|
+
"mock-ssh uses the daemon's config; restart the daemon with the intended config",
|
|
229
233
|
)
|
|
230
234
|
key_dir = key_dir or default_key_dir(host_id)
|
|
231
235
|
ssh_host_alias = ssh_host_alias or default_ssh_host_alias(host_id)
|
|
@@ -233,16 +237,30 @@ async def serve_mock_ssh(
|
|
|
233
237
|
client = HostBridgeClient()
|
|
234
238
|
client.hello()
|
|
235
239
|
session_factory = HostBridgeSessionFactory(client, host_id, connect_timeout)
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
process_factory=HostBridgeSSHProcess(client, session_factory, command_timeout, output_limit),
|
|
243
|
-
sftp_factory=lambda chan: HostBridgeSFTPServer(chan, client, session_factory, command_timeout, output_limit),
|
|
244
|
-
)
|
|
240
|
+
|
|
241
|
+
async def connect_mux_client() -> MuxRpcClient:
|
|
242
|
+
return await MuxRpcClient.connect_unix(client.socket_path)
|
|
243
|
+
|
|
244
|
+
tunnel_connector = TunnelConnector(host_id, connect_mux_client, connect_timeout=connect_timeout)
|
|
245
|
+
server: asyncssh.SSHAcceptor | None = None
|
|
245
246
|
try:
|
|
247
|
+
await tunnel_connector.start()
|
|
248
|
+
server, selected_port = await create_server_on_available_port(
|
|
249
|
+
server_factory=lambda: HostBridgeSSHServer(public_key, tunnel_connector),
|
|
250
|
+
listen_host=listen_host,
|
|
251
|
+
port=port,
|
|
252
|
+
server_host_keys=[str(host_key_path)],
|
|
253
|
+
process_factory=HostBridgeSSHProcess(client, session_factory, command_timeout, output_limit),
|
|
254
|
+
sftp_factory=lambda chan: HostBridgeSFTPServer(
|
|
255
|
+
chan,
|
|
256
|
+
client,
|
|
257
|
+
session_factory,
|
|
258
|
+
command_timeout,
|
|
259
|
+
output_limit,
|
|
260
|
+
DEFAULT_SFTP_MAX_OPEN_HANDLES,
|
|
261
|
+
DEFAULT_SFTP_MAX_TEMP_BYTES,
|
|
262
|
+
),
|
|
263
|
+
)
|
|
246
264
|
installed_config_path: Path | None = None
|
|
247
265
|
if install_ssh_config:
|
|
248
266
|
installed_config_path = install_ssh_config_block(
|
|
@@ -262,11 +280,31 @@ async def serve_mock_ssh(
|
|
|
262
280
|
await server.wait_closed()
|
|
263
281
|
return 0
|
|
264
282
|
finally:
|
|
283
|
+
primary_error = sys.exception()
|
|
284
|
+
cleanup_errors: list[BaseException] = []
|
|
285
|
+
if server is not None:
|
|
286
|
+
try:
|
|
287
|
+
server.close()
|
|
288
|
+
await server.wait_closed()
|
|
289
|
+
except BaseException as exc:
|
|
290
|
+
cleanup_errors.append(exc)
|
|
291
|
+
try:
|
|
292
|
+
await tunnel_connector.close()
|
|
293
|
+
except BaseException as exc:
|
|
294
|
+
cleanup_errors.append(exc)
|
|
295
|
+
try:
|
|
296
|
+
await asyncio.to_thread(session_factory.shutdown)
|
|
297
|
+
except BaseException as exc:
|
|
298
|
+
cleanup_errors.append(exc)
|
|
265
299
|
try:
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
300
|
+
await asyncio.to_thread(client.close)
|
|
301
|
+
except BaseException as exc:
|
|
302
|
+
cleanup_errors.append(exc)
|
|
303
|
+
if cleanup_errors:
|
|
304
|
+
if primary_error is None:
|
|
305
|
+
raise cleanup_errors[0]
|
|
306
|
+
for error in cleanup_errors:
|
|
307
|
+
primary_error.add_note(f"mock-ssh cleanup failed: {error!r}")
|
|
270
308
|
|
|
271
309
|
|
|
272
310
|
def run_mock_ssh(**kwargs: object) -> int:
|
|
@@ -5,12 +5,12 @@ import queue
|
|
|
5
5
|
import sys
|
|
6
6
|
import threading
|
|
7
7
|
from contextlib import suppress
|
|
8
|
-
from functools import partial
|
|
9
8
|
from typing import cast
|
|
10
9
|
|
|
11
10
|
import asyncssh
|
|
12
11
|
|
|
13
|
-
from .
|
|
12
|
+
from .async_lifecycle import finish_task_despite_cancellation
|
|
13
|
+
from .client import ExecResult, HostBridgeClient, HostBridgeError
|
|
14
14
|
from .mock_ssh_session import HostBridgeSessionFactory
|
|
15
15
|
|
|
16
16
|
|
|
@@ -36,12 +36,17 @@ class HostBridgeSSHProcess:
|
|
|
36
36
|
except asyncio.CancelledError:
|
|
37
37
|
session_id = None
|
|
38
38
|
with suppress(BaseException):
|
|
39
|
-
session_id = await open_task
|
|
39
|
+
session_id = await finish_task_despite_cancellation(open_task)
|
|
40
40
|
if session_id is not None:
|
|
41
|
-
|
|
41
|
+
try:
|
|
42
|
+
await self._close_session(session_id, reusable=False)
|
|
43
|
+
except Exception as exc:
|
|
44
|
+
self._report_shell_error(process, exc)
|
|
45
|
+
self._set_exit_status(process, 255)
|
|
46
|
+
return
|
|
47
|
+
if session_id is None:
|
|
42
48
|
self._set_exit_status(process, 255)
|
|
43
49
|
return
|
|
44
|
-
assert session_id is not None
|
|
45
50
|
reusable = False
|
|
46
51
|
try:
|
|
47
52
|
reusable = await self._run(process, session_id)
|
|
@@ -49,7 +54,15 @@ class HostBridgeSSHProcess:
|
|
|
49
54
|
self._set_exit_status(process, 255)
|
|
50
55
|
return
|
|
51
56
|
finally:
|
|
52
|
-
|
|
57
|
+
try:
|
|
58
|
+
await self._close_session(session_id, reusable=reusable)
|
|
59
|
+
except Exception as exc:
|
|
60
|
+
self._report_shell_error(process, exc)
|
|
61
|
+
self._set_exit_status(process, 255)
|
|
62
|
+
|
|
63
|
+
async def _close_session(self, session_id: str, *, reusable: bool) -> None:
|
|
64
|
+
close_task = asyncio.create_task(asyncio.to_thread(self._session_factory.close, session_id, reusable=reusable))
|
|
65
|
+
await finish_task_despite_cancellation(close_task)
|
|
53
66
|
|
|
54
67
|
async def _run(self, process: asyncssh.SSHServerProcess[bytes], session_id: str) -> bool:
|
|
55
68
|
command = process.command
|
|
@@ -113,40 +126,84 @@ class HostBridgeSSHProcess:
|
|
|
113
126
|
)
|
|
114
127
|
staging_deadline = loop.time() + self._timeout
|
|
115
128
|
|
|
116
|
-
async def put(item: bytes | BaseException | object) ->
|
|
129
|
+
async def put(item: bytes | BaseException | object) -> bool:
|
|
117
130
|
while True:
|
|
131
|
+
if consumer.done():
|
|
132
|
+
await consumer
|
|
133
|
+
return False
|
|
118
134
|
try:
|
|
119
135
|
items.put_nowait(item)
|
|
120
|
-
return
|
|
136
|
+
return True
|
|
121
137
|
except queue.Full:
|
|
122
|
-
if consumer.done():
|
|
123
|
-
await consumer
|
|
124
138
|
await asyncio.sleep(0.005)
|
|
125
139
|
|
|
126
140
|
try:
|
|
127
141
|
while not process.stdin.at_eof():
|
|
128
142
|
remaining = staging_deadline - loop.time()
|
|
129
143
|
if remaining <= 0:
|
|
130
|
-
await put(HostBridgeError("timeout", "timed out waiting for mock-ssh exec stdin"))
|
|
144
|
+
if not await put(HostBridgeError("timeout", "timed out waiting for mock-ssh exec stdin")):
|
|
145
|
+
return await consumer
|
|
131
146
|
break
|
|
147
|
+
read_task = asyncio.create_task(process.stdin.read(64 * 1024))
|
|
132
148
|
try:
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
149
|
+
completed, _ = await asyncio.wait(
|
|
150
|
+
(read_task, consumer),
|
|
151
|
+
timeout=remaining,
|
|
152
|
+
return_when=asyncio.FIRST_COMPLETED,
|
|
153
|
+
)
|
|
154
|
+
except BaseException:
|
|
155
|
+
read_task.cancel()
|
|
156
|
+
await asyncio.gather(read_task, return_exceptions=True)
|
|
157
|
+
raise
|
|
158
|
+
if consumer in completed:
|
|
159
|
+
read_task.cancel()
|
|
160
|
+
await asyncio.gather(read_task, return_exceptions=True)
|
|
161
|
+
return await consumer
|
|
162
|
+
if read_task not in completed:
|
|
163
|
+
read_task.cancel()
|
|
164
|
+
await asyncio.gather(read_task, return_exceptions=True)
|
|
165
|
+
if not await put(HostBridgeError("timeout", "timed out waiting for mock-ssh exec stdin")):
|
|
166
|
+
return await consumer
|
|
136
167
|
break
|
|
168
|
+
try:
|
|
169
|
+
data = read_task.result()
|
|
170
|
+
except asyncio.CancelledError:
|
|
171
|
+
raise
|
|
137
172
|
except BaseException as exc:
|
|
138
|
-
await put(exc)
|
|
173
|
+
if not await put(exc):
|
|
174
|
+
return await consumer
|
|
139
175
|
break
|
|
140
176
|
if not data:
|
|
141
177
|
break
|
|
142
|
-
await put(_to_bytes(data))
|
|
143
|
-
|
|
144
|
-
|
|
178
|
+
if not await put(_to_bytes(data)):
|
|
179
|
+
return await consumer
|
|
180
|
+
if not await put(end):
|
|
181
|
+
return await consumer
|
|
182
|
+
return await asyncio.shield(consumer)
|
|
145
183
|
except asyncio.CancelledError:
|
|
146
184
|
cancelled.set()
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
185
|
+
close_succeeded = False
|
|
186
|
+
|
|
187
|
+
def observe_consumer(future: asyncio.Future[ExecResult]) -> None:
|
|
188
|
+
try:
|
|
189
|
+
future.result()
|
|
190
|
+
except asyncio.CancelledError:
|
|
191
|
+
return
|
|
192
|
+
except Exception as exc:
|
|
193
|
+
self._report_shell_error(process, exc)
|
|
194
|
+
|
|
195
|
+
try:
|
|
196
|
+
await self._close_session(session_id, reusable=False)
|
|
197
|
+
close_succeeded = True
|
|
198
|
+
except Exception as exc:
|
|
199
|
+
self._report_shell_error(process, exc)
|
|
200
|
+
with suppress(queue.Full):
|
|
201
|
+
items.put_nowait(end)
|
|
202
|
+
if close_succeeded:
|
|
203
|
+
with suppress(BaseException):
|
|
204
|
+
await finish_task_despite_cancellation(consumer)
|
|
205
|
+
else:
|
|
206
|
+
consumer.add_done_callback(observe_consumer)
|
|
150
207
|
raise
|
|
151
208
|
|
|
152
209
|
@staticmethod
|
|
@@ -155,56 +212,88 @@ class HostBridgeSSHProcess:
|
|
|
155
212
|
process.exit(status)
|
|
156
213
|
|
|
157
214
|
async def _run_shell(self, process: asyncssh.SSHServerProcess[bytes], session_id: str) -> None:
|
|
158
|
-
loop = asyncio.get_running_loop()
|
|
159
215
|
stop = asyncio.Event()
|
|
160
|
-
|
|
216
|
+
failed = False
|
|
217
|
+
terminal_failed = False
|
|
218
|
+
exit_code: int | None = None
|
|
219
|
+
exit_signal: str | None = None
|
|
220
|
+
open_task = asyncio.create_task(asyncio.to_thread(self._client.open_shell, session_id, owner="mock-ssh"))
|
|
221
|
+
try:
|
|
222
|
+
shell = await asyncio.shield(open_task)
|
|
223
|
+
except asyncio.CancelledError:
|
|
224
|
+
with suppress(BaseException):
|
|
225
|
+
shell = await finish_task_despite_cancellation(open_task)
|
|
226
|
+
close_task = asyncio.create_task(asyncio.to_thread(shell.close))
|
|
227
|
+
await finish_task_despite_cancellation(close_task)
|
|
228
|
+
raise
|
|
161
229
|
|
|
162
230
|
async def pump_input() -> None:
|
|
231
|
+
nonlocal failed
|
|
163
232
|
try:
|
|
164
233
|
while not process.stdin.at_eof():
|
|
165
234
|
data = await process.stdin.read(4096)
|
|
166
235
|
if not data:
|
|
167
236
|
break
|
|
168
|
-
await
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
237
|
+
await asyncio.to_thread(shell.send, _to_bytes(data))
|
|
238
|
+
except Exception as exc:
|
|
239
|
+
failed = True
|
|
240
|
+
self._report_shell_error(process, exc)
|
|
172
241
|
finally:
|
|
173
|
-
|
|
242
|
+
try:
|
|
243
|
+
await asyncio.to_thread(shell.send_eof)
|
|
244
|
+
except Exception as exc:
|
|
245
|
+
failed = True
|
|
246
|
+
self._report_shell_error(process, exc)
|
|
247
|
+
stop.set()
|
|
174
248
|
|
|
175
249
|
async def pump_output() -> None:
|
|
250
|
+
nonlocal exit_code, exit_signal, failed, terminal_failed
|
|
176
251
|
try:
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
lambda: self._client.shell_read(
|
|
182
|
-
session_id,
|
|
183
|
-
timeout=0.2,
|
|
184
|
-
max_bytes=4096,
|
|
185
|
-
owner="mock-ssh",
|
|
186
|
-
),
|
|
187
|
-
)
|
|
188
|
-
if result.data:
|
|
189
|
-
idle_after_input = 0
|
|
190
|
-
process.stdout.write(result.data)
|
|
191
|
-
elif input_done.is_set():
|
|
192
|
-
idle_after_input += 1
|
|
193
|
-
if idle_after_input >= 5:
|
|
194
|
-
stop.set()
|
|
195
|
-
break
|
|
196
|
-
if not result.alive:
|
|
197
|
-
stop.set()
|
|
198
|
-
break
|
|
252
|
+
while data := await asyncio.to_thread(shell.receive):
|
|
253
|
+
process.stdout.write(data)
|
|
254
|
+
exit_code = shell.exit_code
|
|
255
|
+
exit_signal = shell.exit_signal
|
|
199
256
|
except Exception as exc:
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
257
|
+
failed = True
|
|
258
|
+
terminal_failed = True
|
|
259
|
+
self._report_shell_error(process, exc)
|
|
260
|
+
finally:
|
|
203
261
|
stop.set()
|
|
204
262
|
|
|
205
263
|
tasks = [asyncio.create_task(pump_input()), asyncio.create_task(pump_output())]
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
264
|
+
try:
|
|
265
|
+
await stop.wait()
|
|
266
|
+
finally:
|
|
267
|
+
try:
|
|
268
|
+
close_task = asyncio.create_task(asyncio.to_thread(shell.close))
|
|
269
|
+
await finish_task_despite_cancellation(close_task)
|
|
270
|
+
except Exception as exc:
|
|
271
|
+
failed = True
|
|
272
|
+
self._report_shell_error(process, exc)
|
|
273
|
+
finally:
|
|
274
|
+
for task in tasks:
|
|
275
|
+
task.cancel()
|
|
276
|
+
await asyncio.gather(*tasks, return_exceptions=True)
|
|
277
|
+
if terminal_failed:
|
|
278
|
+
self._set_exit_status(process, 255)
|
|
279
|
+
elif failed:
|
|
280
|
+
self._set_exit_status(process, 1)
|
|
281
|
+
elif isinstance(exit_code, int) and not isinstance(exit_code, bool) and 0 <= exit_code <= 255:
|
|
282
|
+
self._set_exit_status(process, exit_code)
|
|
283
|
+
elif isinstance(exit_signal, str) and exit_signal.strip():
|
|
284
|
+
self._set_exit_signal(process, exit_signal)
|
|
285
|
+
else:
|
|
286
|
+
self._set_exit_status(process, 255)
|
|
287
|
+
|
|
288
|
+
@staticmethod
|
|
289
|
+
def _report_shell_error(process: asyncssh.SSHServerProcess[bytes], exc: Exception) -> None:
|
|
290
|
+
print(f"hostbridge mock-ssh shell failed: {exc}", file=sys.stderr, flush=True)
|
|
291
|
+
with suppress(Exception):
|
|
292
|
+
process.stderr.write(f"hostbridge mock-ssh shell error: {exc}\n".encode("utf-8", errors="replace"))
|
|
293
|
+
|
|
294
|
+
@classmethod
|
|
295
|
+
def _set_exit_signal(cls, process: asyncssh.SSHServerProcess[bytes], signal_name: str) -> None:
|
|
296
|
+
try:
|
|
297
|
+
process.exit_with_signal(signal_name)
|
|
298
|
+
except Exception:
|
|
299
|
+
cls._set_exit_status(process, 255)
|