@neoline/hostbridge 2.0.4 → 2.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -9
- package/docs/ARCHITECTURE.md +62 -0
- package/package.json +2 -1
- package/pyproject.toml +1 -1
- package/src/server_control_mcp/__init__.py +4 -3
- package/src/server_control_mcp/async_lifecycle.py +41 -0
- package/src/server_control_mcp/cli.py +1 -1
- package/src/server_control_mcp/client.py +302 -287
- package/src/server_control_mcp/config.py +2 -1
- package/src/server_control_mcp/daemon.py +233 -526
- package/src/server_control_mcp/doctor.py +34 -2
- package/src/server_control_mcp/hosts.py +136 -2
- package/src/server_control_mcp/mock_ssh.py +52 -14
- package/src/server_control_mcp/mock_ssh_exec.py +147 -58
- package/src/server_control_mcp/mock_ssh_session.py +3 -2
- package/src/server_control_mcp/mock_ssh_sftp.py +126 -28
- package/src/server_control_mcp/mock_ssh_tunnel.py +141 -444
- package/src/server_control_mcp/mux_connection.py +326 -0
- package/src/server_control_mcp/mux_daemon.py +186 -0
- package/src/server_control_mcp/mux_protocol.py +279 -0
- package/src/server_control_mcp/mux_records.py +71 -0
- package/src/server_control_mcp/mux_rpc.py +1779 -0
- package/src/server_control_mcp/mux_service.py +506 -0
- package/src/server_control_mcp/mux_stream.py +283 -0
- package/src/server_control_mcp/mux_sync_client.py +224 -0
- package/src/server_control_mcp/remote_agent_bundle.py +56 -0
- package/src/server_control_mcp/remote_mux_agent.py +769 -0
- package/src/server_control_mcp/runtime_services.py +862 -0
- package/src/server_control_mcp/server.py +33 -18
- package/src/server_control_mcp/services.py +2 -666
- package/src/server_control_mcp/transports/__init__.py +4 -8
- package/src/server_control_mcp/transports/base.py +60 -38
- package/src/server_control_mcp/transports/ssh.py +206 -259
- package/src/server_control_mcp/tunnel_manager.py +1245 -0
- package/src/server_control_mcp/tunnel_native_ssh.py +454 -0
- package/src/server_control_mcp/tunnel_providers.py +20 -0
- package/src/server_control_mcp/tunnel_pty_agent.py +909 -0
- package/src/server_control_mcp/protocol.py +0 -363
- package/src/server_control_mcp/remote_tunnel_agent.py +0 -143
- package/src/server_control_mcp/transports/pty.py +0 -515
|
@@ -1,29 +1,9 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""Shared service errors and result models."""
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
-
import
|
|
6
|
-
import logging
|
|
7
|
-
import re
|
|
8
|
-
import threading
|
|
9
|
-
import time
|
|
10
|
-
import uuid
|
|
11
|
-
from collections.abc import Callable, Iterable, Iterator
|
|
12
|
-
from contextlib import contextmanager, suppress
|
|
13
|
-
from dataclasses import dataclass, field
|
|
5
|
+
from dataclasses import dataclass
|
|
14
6
|
|
|
15
|
-
from .hosts import HostConfig, HostRegistry
|
|
16
|
-
from .policy import AuditSink, PolicyConfig, audit_event
|
|
17
|
-
from .remote_task_agent import (
|
|
18
|
-
build_remote_task_command,
|
|
19
|
-
build_remote_tasks_cleanup_command,
|
|
20
|
-
remote_job_dir,
|
|
21
|
-
remote_job_dir_assignment,
|
|
22
|
-
)
|
|
23
|
-
from .transports.base import ExecResult, ShellReadResult, TransferResult, Transport, TransportBusy, TransportError
|
|
24
|
-
|
|
25
|
-
TransportFactory = Callable[[HostConfig, float], Transport]
|
|
26
|
-
logger = logging.getLogger(__name__)
|
|
27
7
|
MAX_EXEC_OUTPUT_BYTES = 3 * 1024 * 1024 - 64 * 1024
|
|
28
8
|
|
|
29
9
|
|
|
@@ -42,33 +22,6 @@ class ServiceError(RuntimeError):
|
|
|
42
22
|
self.details = details or {}
|
|
43
23
|
|
|
44
24
|
|
|
45
|
-
@dataclass(slots=True)
|
|
46
|
-
class ServiceSession:
|
|
47
|
-
session_id: str
|
|
48
|
-
host_id: str
|
|
49
|
-
owner: str | None
|
|
50
|
-
transport: Transport = field(repr=False)
|
|
51
|
-
created_at: float = field(default_factory=time.time)
|
|
52
|
-
last_used_at: float = field(default_factory=time.time)
|
|
53
|
-
active_operations: int = 0
|
|
54
|
-
closing: bool = False
|
|
55
|
-
|
|
56
|
-
def describe(self) -> dict[str, object]:
|
|
57
|
-
return {
|
|
58
|
-
"session_id": self.session_id,
|
|
59
|
-
"host_id": self.host_id,
|
|
60
|
-
"owner": self.owner,
|
|
61
|
-
"created_at": self.created_at,
|
|
62
|
-
"last_used_at": self.last_used_at,
|
|
63
|
-
"active_operations": self.active_operations,
|
|
64
|
-
"capabilities": {
|
|
65
|
-
"separate_stderr": self.transport.capabilities.separate_stderr,
|
|
66
|
-
"native_binary_transfer": self.transport.capabilities.native_binary_transfer,
|
|
67
|
-
"parallel_channels": self.transport.capabilities.parallel_channels,
|
|
68
|
-
},
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
|
|
72
25
|
@dataclass(frozen=True, slots=True)
|
|
73
26
|
class ServiceExecResult:
|
|
74
27
|
session_id: str
|
|
@@ -87,620 +40,3 @@ class ServiceTask:
|
|
|
87
40
|
job_dir: str
|
|
88
41
|
remote_pid: int | None
|
|
89
42
|
owner: str | None
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
class HostBridgeServices:
|
|
93
|
-
def __init__(
|
|
94
|
-
self,
|
|
95
|
-
hosts: HostRegistry,
|
|
96
|
-
*,
|
|
97
|
-
transport_factory: TransportFactory,
|
|
98
|
-
default_connect_timeout: float = 30,
|
|
99
|
-
policy: PolicyConfig | None = None,
|
|
100
|
-
audit_sink: AuditSink | None = None,
|
|
101
|
-
max_tasks_per_session: int = 32,
|
|
102
|
-
) -> None:
|
|
103
|
-
if max_tasks_per_session <= 0:
|
|
104
|
-
raise ValueError("max_tasks_per_session must be positive")
|
|
105
|
-
self.hosts = hosts
|
|
106
|
-
self._transport_factory = transport_factory
|
|
107
|
-
self._default_connect_timeout = default_connect_timeout
|
|
108
|
-
self._policy = policy
|
|
109
|
-
self._audit = audit_sink
|
|
110
|
-
self._max_tasks_per_session = max_tasks_per_session
|
|
111
|
-
self._sessions: dict[str, ServiceSession] = {}
|
|
112
|
-
self._pending_sessions: dict[str, int] = {}
|
|
113
|
-
self._tasks: dict[str, ServiceTask] = {}
|
|
114
|
-
self._launching_tasks: dict[str, ServiceTask] = {}
|
|
115
|
-
self._sessions_lock = threading.RLock()
|
|
116
|
-
self._operation_local = threading.local()
|
|
117
|
-
self._closing = False
|
|
118
|
-
self._reaper_stop = threading.Event()
|
|
119
|
-
self._reaper_thread: threading.Thread | None = None
|
|
120
|
-
|
|
121
|
-
def open_session(
|
|
122
|
-
self,
|
|
123
|
-
host_id: str,
|
|
124
|
-
*,
|
|
125
|
-
owner: str | None = None,
|
|
126
|
-
connect_timeout: float | None = None,
|
|
127
|
-
) -> ServiceSession:
|
|
128
|
-
try:
|
|
129
|
-
host = self.hosts.get(host_id)
|
|
130
|
-
except KeyError as exc:
|
|
131
|
-
raise ServiceError("not_found", str(exc)) from exc
|
|
132
|
-
with self._sessions_lock:
|
|
133
|
-
if self._closing:
|
|
134
|
-
raise ServiceError("connection_lost", "HostBridge services are shutting down", retryable=True)
|
|
135
|
-
current = sum(session.host_id == host.id for session in self._sessions.values())
|
|
136
|
-
pending = self._pending_sessions.get(host.id, 0)
|
|
137
|
-
if current + pending >= host.limits.max_sessions:
|
|
138
|
-
raise ServiceError("resource_limit", f"host {host_id} reached its session limit")
|
|
139
|
-
self._pending_sessions[host.id] = pending + 1
|
|
140
|
-
effective_timeout = self._default_connect_timeout if connect_timeout is None else connect_timeout
|
|
141
|
-
if effective_timeout <= 0:
|
|
142
|
-
self._release_pending_session(host.id)
|
|
143
|
-
raise ServiceError("invalid_request", "connect_timeout must be positive")
|
|
144
|
-
try:
|
|
145
|
-
transport = self._transport_factory(host, effective_timeout)
|
|
146
|
-
except BaseException as exc:
|
|
147
|
-
self._release_pending_session(host.id)
|
|
148
|
-
if not isinstance(exc, Exception):
|
|
149
|
-
raise
|
|
150
|
-
raise ServiceError("connection_lost", f"failed to connect to host {host_id}", retryable=True) from exc
|
|
151
|
-
session = ServiceSession(uuid.uuid4().hex, host.id, owner, transport)
|
|
152
|
-
with self._sessions_lock:
|
|
153
|
-
self._release_pending_session_locked(host.id)
|
|
154
|
-
accepted = not self._closing
|
|
155
|
-
if accepted:
|
|
156
|
-
self._sessions[session.session_id] = session
|
|
157
|
-
if not accepted:
|
|
158
|
-
with suppress(Exception):
|
|
159
|
-
transport.close()
|
|
160
|
-
raise ServiceError("connection_lost", "HostBridge services are shutting down", retryable=True)
|
|
161
|
-
self._emit("session_opened", session_id=session.session_id, host_id=host.id, owner_agent_id=owner, outcome="ok")
|
|
162
|
-
return session
|
|
163
|
-
|
|
164
|
-
def list_sessions(self) -> list[dict[str, object]]:
|
|
165
|
-
with self._sessions_lock:
|
|
166
|
-
return [session.describe() for session in self._sessions.values()]
|
|
167
|
-
|
|
168
|
-
def exec(
|
|
169
|
-
self,
|
|
170
|
-
session_id: str,
|
|
171
|
-
command: str,
|
|
172
|
-
*,
|
|
173
|
-
stdin: Iterable[bytes] | None = None,
|
|
174
|
-
timeout: float = 60,
|
|
175
|
-
output_limit: int = 1_000_000,
|
|
176
|
-
owner: str | None = None,
|
|
177
|
-
) -> ServiceExecResult:
|
|
178
|
-
if not command.strip():
|
|
179
|
-
raise ServiceError("invalid_request", "command must not be empty")
|
|
180
|
-
if self._policy is not None:
|
|
181
|
-
decision = self._policy.evaluate(command)
|
|
182
|
-
if not decision.allowed:
|
|
183
|
-
raise ServiceError("denied", decision.reason, details={"matched_pattern": decision.matched_pattern})
|
|
184
|
-
with self._operation(session_id, owner) as session:
|
|
185
|
-
limits = self.hosts.get(session.host_id).limits
|
|
186
|
-
timeout = min(timeout, limits.command_timeout_seconds)
|
|
187
|
-
output_limit = min(output_limit, limits.max_output_bytes, MAX_EXEC_OUTPUT_BYTES)
|
|
188
|
-
stdin = self._bounded_chunks(stdin, limits.max_stdin_bytes, "command stdin")
|
|
189
|
-
try:
|
|
190
|
-
result: ExecResult = session.transport.exec(
|
|
191
|
-
command,
|
|
192
|
-
stdin=stdin,
|
|
193
|
-
timeout=timeout,
|
|
194
|
-
output_limit=output_limit,
|
|
195
|
-
)
|
|
196
|
-
except TransportBusy as exc:
|
|
197
|
-
raise ServiceError("busy", str(exc), retryable=True) from exc
|
|
198
|
-
except TransportError as exc:
|
|
199
|
-
raise ServiceError("remote_error", str(exc), retryable=True) from exc
|
|
200
|
-
if result.timed_out:
|
|
201
|
-
self.close_session(session_id, force=True)
|
|
202
|
-
self._emit(
|
|
203
|
-
"command_run",
|
|
204
|
-
session_id=session.session_id,
|
|
205
|
-
host_id=session.host_id,
|
|
206
|
-
owner_agent_id=owner,
|
|
207
|
-
command=command,
|
|
208
|
-
outcome="timeout" if result.timed_out else f"exit={result.exit_code}",
|
|
209
|
-
)
|
|
210
|
-
return ServiceExecResult(
|
|
211
|
-
session.session_id,
|
|
212
|
-
result.exit_code,
|
|
213
|
-
*self._limit_combined_output(result.stdout, result.stderr, output_limit),
|
|
214
|
-
result.timed_out,
|
|
215
|
-
)
|
|
216
|
-
|
|
217
|
-
def close_session(self, session_id: str, *, owner: str | None = None, force: bool = False) -> None:
|
|
218
|
-
with self._sessions_lock:
|
|
219
|
-
session = self._sessions.get(session_id)
|
|
220
|
-
if session is None:
|
|
221
|
-
raise ServiceError("not_found", f"unknown session {session_id}")
|
|
222
|
-
if not force:
|
|
223
|
-
self._check_owner(session, owner)
|
|
224
|
-
if session.active_operations:
|
|
225
|
-
raise ServiceError("busy", f"session {session_id} has active operations", retryable=True)
|
|
226
|
-
if session.closing:
|
|
227
|
-
raise ServiceError("busy", f"session {session_id} is already closing", retryable=True)
|
|
228
|
-
session.closing = True
|
|
229
|
-
tasks = [
|
|
230
|
-
task
|
|
231
|
-
for task in (*self._tasks.values(), *self._launching_tasks.values())
|
|
232
|
-
if task.session_id == session_id
|
|
233
|
-
]
|
|
234
|
-
try:
|
|
235
|
-
self._cleanup_tasks(session, tasks)
|
|
236
|
-
except Exception as exc:
|
|
237
|
-
if not force:
|
|
238
|
-
with self._sessions_lock:
|
|
239
|
-
session.closing = False
|
|
240
|
-
raise ServiceError("connection_lost", f"failed to close session {session_id}", retryable=True) from exc
|
|
241
|
-
logger.exception("failed to clean up tasks while force-closing HostBridge session %s", session_id)
|
|
242
|
-
try:
|
|
243
|
-
session.transport.close()
|
|
244
|
-
except Exception as exc:
|
|
245
|
-
with self._sessions_lock:
|
|
246
|
-
session.closing = False
|
|
247
|
-
raise ServiceError("connection_lost", f"failed to close session {session_id}", retryable=True) from exc
|
|
248
|
-
with self._sessions_lock:
|
|
249
|
-
self._sessions.pop(session_id, None)
|
|
250
|
-
self._tasks = {task_id: task for task_id, task in self._tasks.items() if task.session_id != session_id}
|
|
251
|
-
self._launching_tasks = {
|
|
252
|
-
task_id: task for task_id, task in self._launching_tasks.items() if task.session_id != session_id
|
|
253
|
-
}
|
|
254
|
-
self._emit(
|
|
255
|
-
"session_closed",
|
|
256
|
-
session_id=session_id,
|
|
257
|
-
host_id=session.host_id,
|
|
258
|
-
owner_agent_id=owner,
|
|
259
|
-
outcome="forced" if force else "user",
|
|
260
|
-
)
|
|
261
|
-
|
|
262
|
-
def close_all(self, *, owner: str | None = None, force: bool = False) -> dict[str, object]:
|
|
263
|
-
with self._sessions_lock:
|
|
264
|
-
if force:
|
|
265
|
-
self._closing = True
|
|
266
|
-
candidates = [session for session in self._sessions.values() if force or session.owner == owner]
|
|
267
|
-
busy = [session.session_id for session in candidates if session.active_operations and not force]
|
|
268
|
-
sessions = [session for session in candidates if force or not session.active_operations]
|
|
269
|
-
closed: list[str] = []
|
|
270
|
-
failed: list[str] = []
|
|
271
|
-
for session in sessions:
|
|
272
|
-
try:
|
|
273
|
-
self.close_session(session.session_id, owner=owner, force=force)
|
|
274
|
-
except ServiceError:
|
|
275
|
-
failed.append(session.session_id)
|
|
276
|
-
logger.exception("failed to close HostBridge session %s", session.session_id)
|
|
277
|
-
else:
|
|
278
|
-
closed.append(session.session_id)
|
|
279
|
-
return {"closed": closed, "count": len(closed), "busy": busy, "failed": failed}
|
|
280
|
-
|
|
281
|
-
def _release_pending_session(self, host_id: str) -> None:
|
|
282
|
-
with self._sessions_lock:
|
|
283
|
-
self._release_pending_session_locked(host_id)
|
|
284
|
-
|
|
285
|
-
def _release_pending_session_locked(self, host_id: str) -> None:
|
|
286
|
-
pending = self._pending_sessions.get(host_id, 0)
|
|
287
|
-
if pending <= 1:
|
|
288
|
-
self._pending_sessions.pop(host_id, None)
|
|
289
|
-
else:
|
|
290
|
-
self._pending_sessions[host_id] = pending - 1
|
|
291
|
-
|
|
292
|
-
def reap_idle(self, timeout: float) -> int:
|
|
293
|
-
if timeout <= 0:
|
|
294
|
-
return 0
|
|
295
|
-
now = time.time()
|
|
296
|
-
with self._sessions_lock:
|
|
297
|
-
expired = [
|
|
298
|
-
session
|
|
299
|
-
for session in self._sessions.values()
|
|
300
|
-
if session.active_operations == 0 and now - session.last_used_at > timeout
|
|
301
|
-
]
|
|
302
|
-
closed = 0
|
|
303
|
-
for session in expired:
|
|
304
|
-
try:
|
|
305
|
-
self.close_session(session.session_id, force=True)
|
|
306
|
-
except ServiceError:
|
|
307
|
-
logger.exception("failed to reap idle HostBridge session %s", session.session_id)
|
|
308
|
-
else:
|
|
309
|
-
closed += 1
|
|
310
|
-
self._emit(
|
|
311
|
-
"session_idle_closed", session_id=session.session_id, host_id=session.host_id, outcome="expired"
|
|
312
|
-
)
|
|
313
|
-
return closed
|
|
314
|
-
|
|
315
|
-
def start_idle_reaper(self, *, timeout: float, interval: float = 30) -> None:
|
|
316
|
-
if timeout <= 0 or interval <= 0:
|
|
317
|
-
return
|
|
318
|
-
if self._reaper_thread is not None and self._reaper_thread.is_alive():
|
|
319
|
-
return
|
|
320
|
-
self._reaper_stop.clear()
|
|
321
|
-
|
|
322
|
-
def reap_loop() -> None:
|
|
323
|
-
while not self._reaper_stop.wait(interval):
|
|
324
|
-
self.reap_idle(timeout)
|
|
325
|
-
|
|
326
|
-
self._reaper_thread = threading.Thread(target=reap_loop, daemon=True, name="hostbridge-v1-reaper")
|
|
327
|
-
self._reaper_thread.start()
|
|
328
|
-
|
|
329
|
-
def stop_idle_reaper(self) -> None:
|
|
330
|
-
self._reaper_stop.set()
|
|
331
|
-
thread = self._reaper_thread
|
|
332
|
-
if thread is not None and thread.is_alive():
|
|
333
|
-
thread.join(timeout=2)
|
|
334
|
-
self._reaper_thread = None
|
|
335
|
-
|
|
336
|
-
def start_task(self, session_id: str, command: str, *, owner: str | None = None) -> ServiceTask:
|
|
337
|
-
if not command.strip():
|
|
338
|
-
raise ServiceError("invalid_request", "command must not be empty")
|
|
339
|
-
with self._operation(session_id, owner) as session:
|
|
340
|
-
task_id = uuid.uuid4().hex
|
|
341
|
-
job_dir = remote_job_dir(task_id)
|
|
342
|
-
script = build_remote_task_command(command, task_id)
|
|
343
|
-
launching_task = ServiceTask(task_id, session_id, session.host_id, command, job_dir, None, owner)
|
|
344
|
-
with self._sessions_lock:
|
|
345
|
-
retained = sum(
|
|
346
|
-
task.session_id == session_id for task in (*self._tasks.values(), *self._launching_tasks.values())
|
|
347
|
-
)
|
|
348
|
-
if retained >= self._max_tasks_per_session:
|
|
349
|
-
raise ServiceError("resource_limit", f"session {session_id} reached its task retention limit")
|
|
350
|
-
self._launching_tasks[task_id] = launching_task
|
|
351
|
-
try:
|
|
352
|
-
result = self.exec(session_id, script, timeout=10, output_limit=4096, owner=owner)
|
|
353
|
-
pid_match = re.fullmatch(rb"\s*([1-9]\d*)\s*", result.stdout)
|
|
354
|
-
if result.timed_out or result.exit_code != 0 or pid_match is None:
|
|
355
|
-
raise ServiceError("remote_error", f"failed to launch task {task_id}", retryable=True)
|
|
356
|
-
task = ServiceTask(
|
|
357
|
-
task_id,
|
|
358
|
-
session_id,
|
|
359
|
-
session.host_id,
|
|
360
|
-
command,
|
|
361
|
-
job_dir,
|
|
362
|
-
int(pid_match.group(1)),
|
|
363
|
-
owner,
|
|
364
|
-
)
|
|
365
|
-
with self._sessions_lock:
|
|
366
|
-
self._launching_tasks.pop(task_id, None)
|
|
367
|
-
self._tasks[task_id] = task
|
|
368
|
-
except BaseException:
|
|
369
|
-
with self._sessions_lock:
|
|
370
|
-
needs_cleanup = task_id in self._launching_tasks and session_id in self._sessions
|
|
371
|
-
if needs_cleanup:
|
|
372
|
-
try:
|
|
373
|
-
self._cleanup_tasks(session, [launching_task])
|
|
374
|
-
except Exception:
|
|
375
|
-
logger.exception("failed to clean up task %s after launch failure", task_id)
|
|
376
|
-
else:
|
|
377
|
-
with self._sessions_lock:
|
|
378
|
-
self._launching_tasks.pop(task_id, None)
|
|
379
|
-
raise
|
|
380
|
-
self._emit(
|
|
381
|
-
"task_started",
|
|
382
|
-
session_id=session_id,
|
|
383
|
-
task_id=task_id,
|
|
384
|
-
host_id=session.host_id,
|
|
385
|
-
owner_agent_id=owner,
|
|
386
|
-
command=command,
|
|
387
|
-
outcome="ok",
|
|
388
|
-
)
|
|
389
|
-
return task
|
|
390
|
-
|
|
391
|
-
def task_status(
|
|
392
|
-
self,
|
|
393
|
-
session_id: str,
|
|
394
|
-
task_id: str,
|
|
395
|
-
*,
|
|
396
|
-
owner: str | None = None,
|
|
397
|
-
tail_lines: int = 80,
|
|
398
|
-
) -> dict[str, object]:
|
|
399
|
-
with self._operation(session_id, owner):
|
|
400
|
-
task = self._get_task(session_id, task_id, owner)
|
|
401
|
-
safe_tail = max(1, min(int(tail_lines), 500))
|
|
402
|
-
script = (
|
|
403
|
-
f'{remote_job_dir_assignment(task.task_id)}; pid=$(cat "$job_dir/pid" 2>/dev/null || true); '
|
|
404
|
-
'if [ -f "$job_dir/exit_code" ]; then state=finished; exit_code=$(cat "$job_dir/exit_code"); '
|
|
405
|
-
'elif [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then state=running; exit_code=\'\'; '
|
|
406
|
-
"else state=unknown; exit_code=''; fi; "
|
|
407
|
-
'printf \'STATE=%s\nPID=%s\nEXIT=%s\n---LOG---\n\' "$state" "$pid" "$exit_code"; '
|
|
408
|
-
f'tail -n {safe_tail} "$job_dir/stdout.log" 2>/dev/null || true'
|
|
409
|
-
)
|
|
410
|
-
result = self.exec(session_id, script, timeout=15, output_limit=1_000_000, owner=owner)
|
|
411
|
-
metadata, _, log = result.stdout.decode("utf-8", errors="replace").partition("---LOG---\n")
|
|
412
|
-
fields: dict[str, str] = {}
|
|
413
|
-
for line in metadata.splitlines():
|
|
414
|
-
key, separator, value = line.partition("=")
|
|
415
|
-
if separator:
|
|
416
|
-
fields[key] = value
|
|
417
|
-
exit_text = fields.get("EXIT", "")
|
|
418
|
-
return {
|
|
419
|
-
"task_id": task.task_id,
|
|
420
|
-
"session_id": session_id,
|
|
421
|
-
"host_id": task.host_id,
|
|
422
|
-
"command": task.command,
|
|
423
|
-
"job_dir": task.job_dir,
|
|
424
|
-
"state": fields.get("STATE", "unknown"),
|
|
425
|
-
"pid": fields.get("PID") or task.remote_pid,
|
|
426
|
-
"exit_code": int(exit_text) if exit_text.isdigit() else None,
|
|
427
|
-
"log_tail": log,
|
|
428
|
-
}
|
|
429
|
-
|
|
430
|
-
def cancel_task(self, session_id: str, task_id: str, *, owner: str | None = None) -> dict[str, object]:
|
|
431
|
-
with self._operation(session_id, owner):
|
|
432
|
-
task = self._get_task(session_id, task_id, owner)
|
|
433
|
-
if task.remote_pid is None:
|
|
434
|
-
raise ServiceError("remote_error", f"task {task_id} has no recorded remote pid")
|
|
435
|
-
script = (
|
|
436
|
-
f"{remote_job_dir_assignment(task.task_id)}; "
|
|
437
|
-
'if [ -f "$job_dir/exit_code" ]; then printf already; '
|
|
438
|
-
f"elif kill -TERM -- -{task.remote_pid} 2>/dev/null; then "
|
|
439
|
-
'attempt=0; while kill -0 -- -$(cat "$job_dir/pid") 2>/dev/null && [ "$attempt" -lt 20 ]; do '
|
|
440
|
-
"attempt=$((attempt + 1)); sleep 0.05; done; "
|
|
441
|
-
'if kill -0 -- -$(cat "$job_dir/pid") 2>/dev/null; then '
|
|
442
|
-
'kill -KILL -- -$(cat "$job_dir/pid") 2>/dev/null || true; fi; '
|
|
443
|
-
'printf \'143\n\' > "$job_dir/.exit_code.tmp"; mv "$job_dir/.exit_code.tmp" "$job_dir/exit_code"; '
|
|
444
|
-
"printf killed; else printf noop; fi"
|
|
445
|
-
)
|
|
446
|
-
result = self.exec(session_id, script, timeout=10, output_limit=4096, owner=owner)
|
|
447
|
-
outcome = (
|
|
448
|
-
result.stdout.decode("utf-8", errors="replace").strip().splitlines()[-1]
|
|
449
|
-
if result.stdout.strip()
|
|
450
|
-
else "unknown"
|
|
451
|
-
)
|
|
452
|
-
self._emit("task_cancelled", session_id=session_id, task_id=task_id, outcome=outcome)
|
|
453
|
-
return {"task_id": task_id, "stopped": outcome in {"killed", "already"}, "outcome": outcome}
|
|
454
|
-
|
|
455
|
-
def upload(
|
|
456
|
-
self,
|
|
457
|
-
session_id: str,
|
|
458
|
-
chunks: Iterable[bytes],
|
|
459
|
-
remote_path: str,
|
|
460
|
-
*,
|
|
461
|
-
mode: int,
|
|
462
|
-
expected_size: int | None,
|
|
463
|
-
expected_sha256: str | None,
|
|
464
|
-
owner: str | None = None,
|
|
465
|
-
) -> TransferResult:
|
|
466
|
-
with self._operation(session_id, owner) as session:
|
|
467
|
-
limit = self.hosts.get(session.host_id).limits.max_transfer_bytes
|
|
468
|
-
if expected_size is not None and expected_size > limit:
|
|
469
|
-
raise ServiceError("resource_limit", f"upload exceeds {limit} bytes")
|
|
470
|
-
bounded_chunks = self._bounded_chunks(chunks, limit, "upload")
|
|
471
|
-
assert bounded_chunks is not None
|
|
472
|
-
try:
|
|
473
|
-
result = session.transport.upload(
|
|
474
|
-
bounded_chunks,
|
|
475
|
-
remote_path,
|
|
476
|
-
mode=mode,
|
|
477
|
-
expected_size=expected_size,
|
|
478
|
-
)
|
|
479
|
-
except TransportError as exc:
|
|
480
|
-
raise ServiceError("remote_error", str(exc), retryable=True) from exc
|
|
481
|
-
if expected_size is not None and result.bytes_transferred != expected_size:
|
|
482
|
-
raise ServiceError(
|
|
483
|
-
"integrity_error",
|
|
484
|
-
f"transfer size mismatch: expected {expected_size}, received {result.bytes_transferred}",
|
|
485
|
-
)
|
|
486
|
-
if expected_sha256 is not None and result.sha256.lower() != expected_sha256.lower():
|
|
487
|
-
raise ServiceError("integrity_error", "transfer SHA-256 mismatch")
|
|
488
|
-
self._emit(
|
|
489
|
-
"file_uploaded",
|
|
490
|
-
session_id=session_id,
|
|
491
|
-
host_id=session.host_id,
|
|
492
|
-
owner_agent_id=owner,
|
|
493
|
-
command=remote_path,
|
|
494
|
-
outcome=f"bytes={result.bytes_transferred}",
|
|
495
|
-
)
|
|
496
|
-
return result
|
|
497
|
-
|
|
498
|
-
def download(
|
|
499
|
-
self,
|
|
500
|
-
session_id: str,
|
|
501
|
-
remote_path: str,
|
|
502
|
-
*,
|
|
503
|
-
chunk_size: int,
|
|
504
|
-
owner: str | None = None,
|
|
505
|
-
) -> Iterator[bytes]:
|
|
506
|
-
def stream() -> Iterator[bytes]:
|
|
507
|
-
with self._operation(session_id, owner) as session:
|
|
508
|
-
limit = self.hosts.get(session.host_id).limits.max_transfer_bytes
|
|
509
|
-
transferred = 0
|
|
510
|
-
try:
|
|
511
|
-
for chunk in session.transport.download(remote_path, chunk_size=chunk_size):
|
|
512
|
-
if not isinstance(chunk, bytes):
|
|
513
|
-
raise ServiceError("internal", "transport returned a non-bytes download chunk")
|
|
514
|
-
transferred += len(chunk)
|
|
515
|
-
if transferred > limit:
|
|
516
|
-
raise ServiceError("resource_limit", f"download exceeds {limit} bytes")
|
|
517
|
-
yield chunk
|
|
518
|
-
except TransportError as exc:
|
|
519
|
-
raise ServiceError("remote_error", str(exc), retryable=True) from exc
|
|
520
|
-
finally:
|
|
521
|
-
self._emit(
|
|
522
|
-
"file_downloaded",
|
|
523
|
-
session_id=session_id,
|
|
524
|
-
host_id=session.host_id,
|
|
525
|
-
owner_agent_id=owner,
|
|
526
|
-
command=remote_path,
|
|
527
|
-
outcome=f"bytes={transferred}",
|
|
528
|
-
)
|
|
529
|
-
|
|
530
|
-
return stream()
|
|
531
|
-
|
|
532
|
-
def write_text(
|
|
533
|
-
self,
|
|
534
|
-
session_id: str,
|
|
535
|
-
remote_path: str,
|
|
536
|
-
content: str,
|
|
537
|
-
*,
|
|
538
|
-
mode: int = 0o644,
|
|
539
|
-
max_bytes: int = 4 * 1024 * 1024,
|
|
540
|
-
owner: str | None = None,
|
|
541
|
-
) -> TransferResult:
|
|
542
|
-
data = content.encode("utf-8")
|
|
543
|
-
session = self._get_session(session_id)
|
|
544
|
-
self._check_owner(session, owner)
|
|
545
|
-
limit = min(max_bytes, self.hosts.get(session.host_id).limits.max_text_bytes)
|
|
546
|
-
if len(data) > limit:
|
|
547
|
-
raise ServiceError("resource_limit", f"text content exceeds {limit} bytes")
|
|
548
|
-
return self.upload(
|
|
549
|
-
session_id,
|
|
550
|
-
iter([data]),
|
|
551
|
-
remote_path,
|
|
552
|
-
mode=mode,
|
|
553
|
-
expected_size=len(data),
|
|
554
|
-
expected_sha256=hashlib.sha256(data).hexdigest(),
|
|
555
|
-
owner=owner,
|
|
556
|
-
)
|
|
557
|
-
|
|
558
|
-
def read_text(
|
|
559
|
-
self,
|
|
560
|
-
session_id: str,
|
|
561
|
-
remote_path: str,
|
|
562
|
-
*,
|
|
563
|
-
max_bytes: int = 4 * 1024 * 1024,
|
|
564
|
-
owner: str | None = None,
|
|
565
|
-
) -> str:
|
|
566
|
-
data = bytearray()
|
|
567
|
-
for chunk in self.download(session_id, remote_path, chunk_size=256 * 1024, owner=owner):
|
|
568
|
-
data.extend(chunk)
|
|
569
|
-
session = self._get_session(session_id)
|
|
570
|
-
limit = min(max_bytes, self.hosts.get(session.host_id).limits.max_text_bytes)
|
|
571
|
-
if len(data) > limit:
|
|
572
|
-
raise ServiceError("resource_limit", f"text file exceeds {limit} bytes")
|
|
573
|
-
try:
|
|
574
|
-
return bytes(data).decode("utf-8")
|
|
575
|
-
except UnicodeDecodeError as exc:
|
|
576
|
-
raise ServiceError("invalid_request", f"remote file is not valid UTF-8 at byte {exc.start}") from exc
|
|
577
|
-
|
|
578
|
-
def shell_write(self, session_id: str, data: bytes, *, owner: str | None = None) -> int:
|
|
579
|
-
with self._operation(session_id, owner) as session:
|
|
580
|
-
limit = self.hosts.get(session.host_id).limits.max_shell_bytes
|
|
581
|
-
if len(data) > limit:
|
|
582
|
-
raise ServiceError("resource_limit", f"shell write exceeds {limit} bytes")
|
|
583
|
-
try:
|
|
584
|
-
session.transport.shell_write(data)
|
|
585
|
-
except TransportError as exc:
|
|
586
|
-
raise ServiceError("remote_error", str(exc), retryable=True) from exc
|
|
587
|
-
return len(data)
|
|
588
|
-
|
|
589
|
-
def shell_read(
|
|
590
|
-
self,
|
|
591
|
-
session_id: str,
|
|
592
|
-
*,
|
|
593
|
-
timeout: float,
|
|
594
|
-
max_bytes: int,
|
|
595
|
-
owner: str | None = None,
|
|
596
|
-
) -> ShellReadResult:
|
|
597
|
-
with self._operation(session_id, owner) as session:
|
|
598
|
-
max_bytes = min(max_bytes, self.hosts.get(session.host_id).limits.max_shell_bytes)
|
|
599
|
-
try:
|
|
600
|
-
return session.transport.shell_read(timeout=timeout, max_bytes=max_bytes)
|
|
601
|
-
except TransportError as exc:
|
|
602
|
-
raise ServiceError("remote_error", str(exc), retryable=True) from exc
|
|
603
|
-
|
|
604
|
-
def _get_session(self, session_id: str) -> ServiceSession:
|
|
605
|
-
with self._sessions_lock:
|
|
606
|
-
session = self._sessions.get(session_id)
|
|
607
|
-
if session is None:
|
|
608
|
-
raise ServiceError("not_found", f"unknown session {session_id}")
|
|
609
|
-
return session
|
|
610
|
-
|
|
611
|
-
@staticmethod
|
|
612
|
-
def _cleanup_tasks(session: ServiceSession, tasks: list[ServiceTask]) -> None:
|
|
613
|
-
if not tasks:
|
|
614
|
-
return
|
|
615
|
-
command = build_remote_tasks_cleanup_command((task.task_id, task.remote_pid) for task in tasks)
|
|
616
|
-
result = session.transport.exec(command, stdin=None, timeout=15, output_limit=4096)
|
|
617
|
-
if result.timed_out or result.exit_code != 0:
|
|
618
|
-
raise TransportError(f"failed to clean up tasks for session {session.session_id}")
|
|
619
|
-
|
|
620
|
-
@contextmanager
|
|
621
|
-
def _operation(self, session_id: str, owner: str | None) -> Iterator[ServiceSession]:
|
|
622
|
-
leased_sessions = getattr(self._operation_local, "session_ids", None)
|
|
623
|
-
if leased_sessions is None:
|
|
624
|
-
leased_sessions = set()
|
|
625
|
-
self._operation_local.session_ids = leased_sessions
|
|
626
|
-
if session_id in leased_sessions:
|
|
627
|
-
leased_session = self._get_session(session_id)
|
|
628
|
-
self._check_owner(leased_session, owner)
|
|
629
|
-
yield leased_session
|
|
630
|
-
return
|
|
631
|
-
with self._sessions_lock:
|
|
632
|
-
session = self._sessions.get(session_id)
|
|
633
|
-
if session is None:
|
|
634
|
-
raise ServiceError("not_found", f"unknown session {session_id}")
|
|
635
|
-
self._check_owner(session, owner)
|
|
636
|
-
if session.closing:
|
|
637
|
-
raise ServiceError("busy", f"session {session_id} is closing", retryable=True)
|
|
638
|
-
capacity = max(1, session.transport.capabilities.parallel_channels)
|
|
639
|
-
if session.active_operations >= capacity:
|
|
640
|
-
raise ServiceError("busy", f"session {session_id} reached its channel capacity", retryable=True)
|
|
641
|
-
session.active_operations += 1
|
|
642
|
-
leased_sessions.add(session_id)
|
|
643
|
-
try:
|
|
644
|
-
yield session
|
|
645
|
-
finally:
|
|
646
|
-
with self._sessions_lock:
|
|
647
|
-
session.active_operations -= 1
|
|
648
|
-
session.last_used_at = time.time()
|
|
649
|
-
leased_sessions.remove(session_id)
|
|
650
|
-
|
|
651
|
-
@staticmethod
|
|
652
|
-
def _limit_combined_output(stdout: bytes, stderr: bytes, limit: int) -> tuple[bytes, bytes]:
|
|
653
|
-
if len(stdout) + len(stderr) <= limit:
|
|
654
|
-
return stdout, stderr
|
|
655
|
-
stdout_keep = min(len(stdout), (limit + 1) // 2)
|
|
656
|
-
stderr_keep = min(len(stderr), limit - stdout_keep)
|
|
657
|
-
remaining = limit - stdout_keep - stderr_keep
|
|
658
|
-
if remaining:
|
|
659
|
-
extra_stdout = min(len(stdout) - stdout_keep, remaining)
|
|
660
|
-
stdout_keep += extra_stdout
|
|
661
|
-
stderr_keep += min(len(stderr) - stderr_keep, remaining - extra_stdout)
|
|
662
|
-
return stdout[-stdout_keep:] if stdout_keep else b"", stderr[-stderr_keep:] if stderr_keep else b""
|
|
663
|
-
|
|
664
|
-
@staticmethod
|
|
665
|
-
def _bounded_chunks(chunks: Iterable[bytes] | None, limit: int, label: str) -> Iterator[bytes] | None:
|
|
666
|
-
if chunks is None:
|
|
667
|
-
return None
|
|
668
|
-
|
|
669
|
-
def bounded() -> Iterator[bytes]:
|
|
670
|
-
transferred = 0
|
|
671
|
-
for chunk in chunks:
|
|
672
|
-
if not isinstance(chunk, bytes):
|
|
673
|
-
raise TypeError(f"{label} chunks must be bytes")
|
|
674
|
-
transferred += len(chunk)
|
|
675
|
-
if transferred > limit:
|
|
676
|
-
raise ServiceError("resource_limit", f"{label} exceeds {limit} bytes")
|
|
677
|
-
yield chunk
|
|
678
|
-
|
|
679
|
-
return bounded()
|
|
680
|
-
|
|
681
|
-
def _get_task(self, session_id: str, task_id: str, owner: str | None) -> ServiceTask:
|
|
682
|
-
session = self._get_session(session_id)
|
|
683
|
-
self._check_owner(session, owner)
|
|
684
|
-
with self._sessions_lock:
|
|
685
|
-
task = self._tasks.get(task_id)
|
|
686
|
-
if task is None or task.session_id != session_id:
|
|
687
|
-
raise ServiceError("not_found", f"unknown task {task_id}")
|
|
688
|
-
if task.owner is not None and task.owner != owner:
|
|
689
|
-
raise ServiceError("denied", f"task {task_id} is owned by another agent")
|
|
690
|
-
return task
|
|
691
|
-
|
|
692
|
-
@staticmethod
|
|
693
|
-
def _check_owner(session: ServiceSession, owner: str | None) -> None:
|
|
694
|
-
if session.owner is not None and session.owner != owner:
|
|
695
|
-
raise ServiceError("denied", f"session {session.session_id} is owned by another agent")
|
|
696
|
-
|
|
697
|
-
def _emit(self, event: str, **fields: object) -> None:
|
|
698
|
-
if self._audit is not None:
|
|
699
|
-
try:
|
|
700
|
-
audit_event(
|
|
701
|
-
self._audit,
|
|
702
|
-
event=event,
|
|
703
|
-
**{key: value for key, value in fields.items() if value is not None}, # type: ignore[arg-type]
|
|
704
|
-
)
|
|
705
|
-
except Exception:
|
|
706
|
-
logger.exception("audit event could not be written: %s", event)
|