@neoline/hostbridge 1.0.0 → 1.1.0

package/README.md

@@ -17,10 +17,10 @@ This repo is portable: it ships with no personal server paths or built-in hosts.
 - `task_stop(session_id, task_id)` — terminate a running background job and reap its process.
 - `session_write(session_id, text)` — send a raw line for interactive edge cases.
 - `session_read(session_id, timeout=1.0, max_chars=20000)` — read available raw output after interactive writes.
-- `file_write_text(session_id, remote_path, content)` — write a small UTF-8 text file through the active shell.
-- `file_read_text(session_id, remote_path)` — read a small UTF-8 text file through the active shell.
-- `file_upload(session_id, local_path, remote_path)` — upload a small local file with shell/base64 transfer.
-- `file_download(session_id, remote_path, local_path)` — download a small remote file with shell/base64 transfer.
+- `file_write_text(session_id, remote_path, content)` — write a UTF-8 text file through the active shell, up to 100MiB.
+- `file_read_text(session_id, remote_path)` — read a UTF-8 text file through the active shell, up to 100MiB.
+- `file_upload(session_id, local_path, remote_path)` — upload a local file with chunked shell/base64 transfer, up to 100MiB.
+- `file_download(session_id, remote_path, local_path)` — download a remote file with chunked shell/base64 transfer, up to 100MiB.
 - `session_close(session_id)` — close a persistent session.
 
 Every tool returns a structured payload with a stable `error_code` on failure (`HOST_NOT_FOUND`, `SESSION_NOT_FOUND`, `POLICY_DENIED`, `TASK_NOT_FOUND`, `TIMEOUT`, `INTERNAL`).
@@ -78,14 +78,14 @@ npm run test:node
 
 ## File transfer model
 
-HostBridge file transfer is shell-based because recorded login paths may be JumpServer menus, OTP flows, wrapper scripts, or `docker exec` sessions rather than standard SSH/SFTP connections. The first-class transfer tools therefore use base64 through the already-open shell session and work best for scripts, configs, logs, and small artifacts:
+HostBridge file transfer is shell-based because recorded login paths may be JumpServer menus, OTP flows, wrapper scripts, or `docker exec` sessions rather than standard SSH/SFTP connections. The first-class transfer tools stream chunked base64 through the already-open shell session, verify hashes, and work for text or binary artifacts up to 100MiB:
 
 ```bash
 file_upload(session_id, "/absolute/local/script.py", "/remote/work/script.py")
 file_download(session_id, "/remote/work/result.log", "/absolute/local/result.log")
 ```
 
-Large files are deliberately rejected with guidance to use remote pull instead, for example `curl`, `wget`, `git clone`, `modelscope download`, or another remote-native downloader. HostBridge does not plan a standard SSH/SFTP fast path; the reliable fallback for non-standard login flows is shell/base64, and large model/data movement should happen from the remote host.
+Give file tools local and remote paths; do not paste base64 into tool arguments. Files larger than 100MiB are deliberately rejected with guidance to use remote pull instead, for example `curl`, `wget`, `git clone`, `modelscope download`, or another remote-native downloader. HostBridge does not plan a standard SSH/SFTP fast path; the reliable fallback for non-standard login flows is shell/base64, and large model/data movement should happen from the remote host.
 
 ## Daemon lifecycle
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@neoline/hostbridge",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "description": "HostBridge MCP server for persistent allowlisted SSH and shell sessions.",
   "type": "module",
   "license": "MIT",

package/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "hostbridge"
-version = "1.0.0"
+version = "1.1.0"
 description = "HostBridge MCP server for persistent allowlisted SSH and shell sessions."
 readme = "README.md"
 requires-python = ">=3.11"
@@ -86,4 +86,4 @@ python_version = "3.11"
 ignore_missing_imports = true
 warn_unused_ignores = true
 warn_redundant_casts = true
-files = ["src/server_control_mcp"]
+files = ["src/server_control_mcp"]

package/src/server_control_mcp/__init__.py

@@ -6,7 +6,7 @@ import sys
 from typing import NoReturn
 
 __all__ = ["__version__", "ensure_runtime"]
-__version__ = "1.0.0"
+__version__ = "1.1.0"
 
 # (import_name, pip_name) pairs for runtime dependency probing.
 _REQUIRED_DEPENDENCIES = (

package/src/server_control_mcp/cli.py

@@ -38,8 +38,9 @@ Use HostBridge when the user asks to connect to, inspect, or operate an allowlis
 - Do not run `hostbridge check` before every connection; only use `hostbridge check <host_id>` when `session_open` times out or fails.
 - Do not search shell history, SSH config, or old scripts unless the setup flow lacks required information or the user explicitly asks.
 - Keep long-running work in `task_start`/`task_status`; use `session_close` or `sessions_close_all` when done.
-- Use `file_write_text`/`file_read_text` for small UTF-8 files and `file_upload`/`file_download` for small binary or text artifacts.
-- Do not use HostBridge file tools for large model/data transfers; start a remote pull instead (`curl`, `wget`, `git clone`, `modelscope download`, etc.) and poll it with `task_status`.
+- Use `file_write_text`/`file_read_text` for UTF-8 files and `file_upload`/`file_download` for binary artifacts up to 100MiB; these tools stream through the already-open shell and work across JumpServer/menu/sudo flows where SFTP may not exist.
+- For files larger than 100MiB, do not use HostBridge file tools; start a remote pull instead (`curl`, `wget`, `git clone`, `modelscope download`, etc.) and poll it with `task_status`.
+- Do not ask the user to provide base64 content. Give file tools local/remote paths; HostBridge handles chunking, encoding, hashing, and cleanup internally.
 {HOSTBRIDGE_GUIDANCE_END}
 """
 SUPPORTED_INSTALL_TARGETS = (

package/src/server_control_mcp/daemon.py

@@ -19,6 +19,7 @@ from .hosts import HostRegistry, load_hosts
 from .manager import (
     DEFAULT_COMMAND_TIMEOUT,
     DEFAULT_CONNECT_TIMEOUT,
+    DEFAULT_MAX_TRANSFER_BYTES,
     PolicyDeniedError,
     RemoteCommandError,
     SessionManager,
@@ -173,7 +174,7 @@ class HostBridgeDaemonHandler(StreamRequestHandler):
                         str(body.get("remote_path", "")),
                         str(body.get("content", "")),
                         mode=str(body.get("mode", "0644")),
-                        max_bytes=int(body.get("max_bytes", 1024 * 1024)),
+                        max_bytes=int(body.get("max_bytes", DEFAULT_MAX_TRANSFER_BYTES)),
                         owner_agent_id=owner,
                         token_factory=token_factory,
                     )
@@ -183,7 +184,7 @@ class HostBridgeDaemonHandler(StreamRequestHandler):
                     transfer=self.manager.file_read_text(
                         parts[1],
                         str(body.get("remote_path", "")),
-                        max_bytes=int(body.get("max_bytes", 1024 * 1024)),
+                        max_bytes=int(body.get("max_bytes", DEFAULT_MAX_TRANSFER_BYTES)),
                         owner_agent_id=owner,
                         token_factory=token_factory,
                     )
@@ -195,7 +196,7 @@ class HostBridgeDaemonHandler(StreamRequestHandler):
                         str(body.get("local_path", "")),
                         str(body.get("remote_path", "")),
                         mode=str(body.get("mode", "0644")),
-                        max_bytes=int(body.get("max_bytes", 1024 * 1024)),
+                        max_bytes=int(body.get("max_bytes", DEFAULT_MAX_TRANSFER_BYTES)),
                         owner_agent_id=owner,
                         token_factory=token_factory,
                     )
@@ -206,7 +207,7 @@ class HostBridgeDaemonHandler(StreamRequestHandler):
                         parts[1],
                         str(body.get("remote_path", "")),
                         str(body.get("local_path", "")),
-                        max_bytes=int(body.get("max_bytes", 1024 * 1024)),
+                        max_bytes=int(body.get("max_bytes", DEFAULT_MAX_TRANSFER_BYTES)),
                         owner_agent_id=owner,
                         token_factory=token_factory,
                     )

package/src/server_control_mcp/manager.py

@@ -24,7 +24,10 @@ DEFAULT_REAPER_INTERVAL_SECONDS = 60
 MAX_OUTPUT_CHARS = 120_000
 REMOTE_JOB_ROOT = "$HOME/.hostbridge/jobs"
 REMOTE_SHELL = "bash"
-DEFAULT_MAX_TRANSFER_BYTES = 1024 * 1024
+MAX_SHELL_TRANSFER_BYTES = 100 * 1024 * 1024
+DEFAULT_MAX_TRANSFER_BYTES = MAX_SHELL_TRANSFER_BYTES
+UPLOAD_BASE64_CHUNK_CHARS = 256 * 1024
+DOWNLOAD_CHUNK_BYTES = 5 * 1024 * 1024
 
 class PolicyDeniedError(RuntimeError):
     """Raised when a command is blocked by the configured policy."""
@@ -607,33 +610,96 @@ class SessionManager:
         token = token_factory()
         remote_q = shlex.quote(remote_path)
         mode_q = shlex.quote(mode)
-        heredoc = f"__HOSTBRIDGE_B64_{token}__"
-        script = (
+        tmp_prefix = f"/tmp/hostbridge.upload.{token}"
+        tmp_b64 = f"{tmp_prefix}.b64"
+        tmp_out = f"{tmp_prefix}.out"
+        cleanup_script = f"rm -f {shlex.quote(tmp_b64)} {shlex.quote(tmp_out)}"
+        init_script = (
             "set -e; "
-            f"target={remote_q}; mode={mode_q}; "
+            f"target={remote_q}; "
             'dir=$(dirname "$target"); mkdir -p "$dir"; '
-            'tmp_b64=$(mktemp "${TMPDIR:-/tmp}/hostbridge.XXXXXX.b64"); '
-            'tmp_out=$(mktemp "${TMPDIR:-/tmp}/hostbridge.XXXXXX.out"); '
-            f"cat > \"$tmp_b64\" <<'{heredoc}'\n{b64}\n{heredoc}\n"
-            'base64 -d "$tmp_b64" > "$tmp_out"; '
-            'chmod "$mode" "$tmp_out"; mv "$tmp_out" "$target"; rm -f "$tmp_b64"; '
-            f"printf 'BYTES=%s\\nSHA256=%s\\n' {len(data)} {shlex.quote(sha256)}"
+            f"rm -f {shlex.quote(tmp_b64)} {shlex.quote(tmp_out)}; "
+            f": > {shlex.quote(tmp_b64)}"
         )
-        result = self.run_command(session_id, script, timeout=60, token_factory=lambda: token, owner_agent_id=owner_agent_id)
+        result = self.run_command(session_id, init_script, timeout=60, token_factory=lambda: token, owner_agent_id=owner_agent_id)
         if result.exit_code != 0:
-            raise RemoteCommandError(result.output or f"failed to upload {remote_path}")
+            raise RemoteCommandError(result.output or f"failed to prepare upload for {remote_path}")
+        try:
+            self._stream_base64_to_remote_file(
+                session_id,
+                b64,
+                tmp_b64,
+                token=token,
+                owner_agent_id=owner_agent_id,
+            )
+            finalize_script = (
+                "set -e; "
+                f"target={remote_q}; mode={mode_q}; tmp_b64={shlex.quote(tmp_b64)}; tmp_out={shlex.quote(tmp_out)}; "
+                'base64 -d "$tmp_b64" > "$tmp_out"; '
+                'chmod "$mode" "$tmp_out"; mv "$tmp_out" "$target"; rm -f "$tmp_b64"; '
+                f"printf 'BYTES=%s\\nSHA256=%s\\n' {len(data)} {shlex.quote(sha256)}"
+            )
+            result = self.run_command(
+                session_id,
+                finalize_script,
+                timeout=60,
+                token_factory=lambda: token,
+                owner_agent_id=owner_agent_id,
+            )
+            if result.exit_code != 0:
+                raise RemoteCommandError(result.output or f"failed to finalize upload for {remote_path}")
+        except Exception:
+            with suppress(Exception):
+                self.run_command(session_id, cleanup_script, timeout=10, token_factory=lambda: token, owner_agent_id=owner_agent_id)
+            raise
         payload: dict[str, object] = {
             "session_id": session_id,
             "remote_path": remote_path,
             "bytes": len(data),
             "sha256": sha256,
-            "strategy": "shell_base64",
+            "strategy": "shell_base64_chunked",
         }
         if local_path is not None:
             payload["local_path"] = local_path
         self._emit("file_uploaded", session_id=session_id, command=remote_path, outcome=f"bytes={len(data)}")
         return payload
 
+    def _stream_base64_to_remote_file(
+        self,
+        session_id: str,
+        encoded: str,
+        remote_tmp_b64: str,
+        *,
+        token: str,
+        owner_agent_id: str | None,
+    ) -> None:
+        session = self._get_session(session_id)
+        self._check_owner(session, owner_agent_id)
+        if session.busy:
+            raise RemoteCommandError(f"session {session_id} is busy")
+        if not session.child.isalive():
+            raise RemoteCommandError(f"session {session_id} is not alive")
+
+        heredoc = f"__HOSTBRIDGE_UPLOAD_{token}__"
+        done_pattern = re.compile(rf"__HOSTBRIDGE_UPLOAD_DONE_{re.escape(token)}__:(\d+)")
+        chunk_size = max(4, int(UPLOAD_BASE64_CHUNK_CHARS))
+        session.busy = True
+        try:
+            session.child.send(f"cat >> {shlex.quote(remote_tmp_b64)} <<'{heredoc}'\n")
+            for offset in range(0, len(encoded), chunk_size):
+                session.child.send(encoded[offset : offset + chunk_size])
+            session.child.send(f"\n{heredoc}\nprintf '\\n__HOSTBRIDGE_UPLOAD_DONE_{token}__:%s\\n' \"$?\"\n")
+            session.child.expect(done_pattern, timeout=60)
+            marker = session.child.after
+            text = marker.group(0) if hasattr(marker, "group") else str(marker)
+            match = re.search(r":(\d+)", text)
+            if match is None or int(match.group(1)) != 0:
+                raise RemoteCommandError(f"failed to stream upload chunk for {session_id}")
+        finally:
+            session.busy = False
+            session.last_used_at = time.time()
+            session.last_output = _tail(str(session.child.before))
+
     def _download_bytes(
         self,
         session_id: str,
@@ -646,39 +712,53 @@ class SessionManager:
         self._validate_remote_path(remote_path)
         token = token_factory()
         remote_q = shlex.quote(remote_path)
-        script = (
+        metadata_script = (
             "set -e; "
             f"target={remote_q}; "
             'bytes=$(wc -c < "$target" | tr -d " "); '
             f"if [ \"$bytes\" -gt {int(max_bytes)} ]; then echo 'FILE_TOO_LARGE remote pull required' >&2; exit 73; fi; "
             "if command -v sha256sum >/dev/null 2>&1; then sha=$(sha256sum \"$target\" | awk '{print $1}'); "
             "else sha=$(shasum -a 256 \"$target\" | awk '{print $1}'); fi; "
-            "printf 'BYTES=%s\\nSHA256=%s\\n---BASE64---\\n' \"$bytes\" \"$sha\"; base64 \"$target\""
+            "printf 'BYTES=%s\\nSHA256=%s\\n' \"$bytes\" \"$sha\""
         )
-        output_limit = max(MAX_OUTPUT_CHARS, int(max_bytes) * 2 + 4096)
         result = self.run_command(
             session_id,
-            script,
+            metadata_script,
             timeout=60,
             token_factory=lambda: token,
             owner_agent_id=owner_agent_id,
-            output_limit=output_limit,
         )
         if result.exit_code == 73:
             self._raise_remote_pull_required(max_bytes)
         if result.exit_code != 0:
             raise RemoteCommandError(result.output or f"failed to download {remote_path}")
-        metadata, sep, encoded = result.output.partition("---BASE64---\n")
-        if not sep:
-            raise RemoteCommandError("download response missing base64 marker")
         fields: dict[str, str] = {}
-        for line in metadata.splitlines():
+        for line in result.output.splitlines():
             key, _, value = line.partition("=")
             if key:
                 fields[key] = value
-        data = base64.b64decode("".join(encoded.split()))
-        byte_count = int(fields.get("BYTES") or len(data))
+        byte_count = int(fields.get("BYTES") or 0)
         self._check_transfer_size(byte_count, max_bytes)
+        chunks: list[bytes] = []
+        chunk_size = max(1, int(DOWNLOAD_CHUNK_BYTES))
+        for index in range((byte_count + chunk_size - 1) // chunk_size):
+            chunk_script = (
+                "set -e; "
+                f"dd if={remote_q} bs={chunk_size} skip={index} count=1 2>/dev/null | base64"
+            )
+            output_limit = max(MAX_OUTPUT_CHARS, chunk_size * 2 + 4096)
+            result = self.run_command(
+                session_id,
+                chunk_script,
+                timeout=60,
+                token_factory=lambda: token,
+                owner_agent_id=owner_agent_id,
+                output_limit=output_limit,
+            )
+            if result.exit_code != 0:
+                raise RemoteCommandError(result.output or f"failed to download chunk {index} from {remote_path}")
+            chunks.append(base64.b64decode("".join(result.output.split())))
+        data = b"".join(chunks)
         sha256 = hashlib.sha256(data).hexdigest()
         remote_sha = fields.get("SHA256") or sha256
         if remote_sha != "ignored" and sha256 != remote_sha:
@@ -689,7 +769,7 @@ class SessionManager:
             "remote_path": remote_path,
             "bytes": len(data),
             "sha256": remote_sha,
-            "strategy": "shell_base64",
+            "strategy": "shell_base64_chunked",
             "data_bytes": data,
         }
 

package/src/server_control_mcp/server.py

@@ -12,6 +12,7 @@ from .manager import (
     DEFAULT_COMMAND_TIMEOUT,
     DEFAULT_CONNECT_TIMEOUT,
     DEFAULT_MAX_TRANSFER_BYTES,
+    MAX_SHELL_TRANSFER_BYTES,
     PolicyDeniedError,
     RemoteCommandError,
 )
@@ -66,6 +67,10 @@ def _qid(value: str) -> str:
     return quote(value, safe="")
 
 
+def _safe_transfer_limit(max_bytes: int) -> int:
+    return max(1, min(int(max_bytes), MAX_SHELL_TRANSFER_BYTES))
+
+
 @mcp.tool()
 def hosts_list() -> dict[str, object]:
     """List approved connection targets and local metadata when available."""
@@ -202,9 +207,9 @@ def file_write_text(
     mode: str = "0644",
     max_bytes: int = DEFAULT_MAX_TRANSFER_BYTES,
 ) -> dict[str, object]:
-    """Write a small UTF-8 text file through the active shell session."""
+    """Write a UTF-8 text file through the active shell session; supports up to 100MiB via chunked base64."""
     try:
-        safe_max = max(1, min(int(max_bytes), DEFAULT_MAX_TRANSFER_BYTES))
+        safe_max = _safe_transfer_limit(max_bytes)
         return _daemon_request(
             "POST",
             f"/sessions/{_qid(session_id)}/files",
@@ -216,9 +221,9 @@ def file_write_text(
 
 @mcp.tool()
 def file_read_text(session_id: str, remote_path: str, max_bytes: int = DEFAULT_MAX_TRANSFER_BYTES) -> dict[str, object]:
-    """Read a small UTF-8 text file through the active shell session."""
+    """Read a UTF-8 text file through the active shell session; supports up to 100MiB via chunked base64."""
     try:
-        safe_max = max(1, min(int(max_bytes), DEFAULT_MAX_TRANSFER_BYTES))
+        safe_max = _safe_transfer_limit(max_bytes)
         return _daemon_request(
             "POST",
             f"/sessions/{_qid(session_id)}/files",
@@ -236,9 +241,9 @@ def file_upload(
     mode: str = "0644",
     max_bytes: int = DEFAULT_MAX_TRANSFER_BYTES,
 ) -> dict[str, object]:
-    """Upload a small local file via shell/base64."""
+    """Upload a local file through the active shell session; supports up to 100MiB via chunked base64."""
     try:
-        safe_max = max(1, min(int(max_bytes), DEFAULT_MAX_TRANSFER_BYTES))
+        safe_max = _safe_transfer_limit(max_bytes)
         return _daemon_request(
             "POST",
             f"/sessions/{_qid(session_id)}/files",
@@ -255,9 +260,9 @@ def file_download(
     local_path: str,
     max_bytes: int = DEFAULT_MAX_TRANSFER_BYTES,
 ) -> dict[str, object]:
-    """Download a small remote file via shell/base64."""
+    """Download a remote file through the active shell session; supports up to 100MiB via chunked base64."""
     try:
-        safe_max = max(1, min(int(max_bytes), DEFAULT_MAX_TRANSFER_BYTES))
+        safe_max = _safe_transfer_limit(max_bytes)
         return _daemon_request(
             "POST",
             f"/sessions/{_qid(session_id)}/files",