@zergai/ztc 0.4.0-beta.1 → 0.5.0-beta.1

package/dist/remote-broker/server.py

@@ -0,0 +1,648 @@
+"""Authenticated fixed-operation receiver; requires a read-only execution image."""
+
+import hashlib
+import hmac
+import json
+import logging
+import os
+import re
+import socket
+import stat
+import subprocess
+import sys
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+L = logging.getLogger(__name__)
+MAXIMUM_BYTES = 8 * 1024 * 1024
+MAXIMUM_CONCURRENT_REQUESTS = 16
+INSTALLATION = Path(__file__).resolve().parent
+
+
+class MountTopologyChangedError(ValueError):
+    """A stable operator-facing refusal without request or filesystem contents."""
+
+
+class RequestNotDispatchedError(ValueError):
+    """Preparation failed before the worker could run."""
+
+
+class FilesystemRegion(BaseModel):
+    model_config = ConfigDict(extra="forbid", strict=True, frozen=True)
+    version: Literal[1] = 1
+    device: str = Field(pattern=r"^[0-9]+:[0-9]+$")
+    path: str = Field(min_length=1, max_length=4096)
+
+
+class MountEntry(BaseModel):
+    model_config = ConfigDict(extra="forbid", strict=True, frozen=True)
+    mount_id: int = Field(gt=0)
+    device: str
+    root: str
+    mount_point: str
+
+
+def read_mount_snapshot() -> bytes:
+    with Path("/proc/self/mountinfo").open("rb") as stream:
+        data = stream.read(MAXIMUM_BYTES + 1)
+    if not data or len(data) > MAXIMUM_BYTES:
+        raise ValueError("Host mount topology is empty or exceeds its bound")
+    return data
+
+
+def decode_mount_path(value: str) -> str:
+    if re.search(r"\\(?!040|011|012|134)", value):
+        raise ValueError("Unsupported host mount path escape")
+    path = re.sub(r"\\(040|011|012|134)", lambda match: chr(int(match[1], 8)), value)
+    if not os.path.isabs(path) or os.path.normpath(path) != path or "\x00" in path:
+        raise ValueError("Host mount paths must be canonical and absolute")
+    return path
+
+
+def parse_mount_snapshot(data: bytes) -> dict[int, MountEntry]:
+    entries: dict[int, MountEntry] = {}
+    for line in data.decode("utf-8").splitlines():
+        before, separator, after = line.partition(" - ")
+        fields = before.split()
+        if (
+            not separator
+            or len(fields) < 6
+            or len(after.split()) < 3
+            or not re.fullmatch(r"[0-9]+", fields[0])
+            or not re.fullmatch(r"[0-9]+:[0-9]+", fields[2])
+        ):
+            raise ValueError("Malformed host mount topology")
+        entry = MountEntry(
+            mount_id=int(fields[0]),
+            device=fields[2],
+            root=decode_mount_path(fields[3]),
+            mount_point=decode_mount_path(fields[4]),
+        )
+        if entry.mount_id in entries:
+            raise ValueError("Duplicate host mount identity")
+        entries[entry.mount_id] = entry
+    if not entries:
+        raise ValueError("Host mount topology is empty")
+    return entries
+
+
+class Endpoint(BaseModel):
+    model_config = ConfigDict(extra="forbid", strict=True, frozen=True)
+    host: str = Field(min_length=1, max_length=253)
+    user: str = Field(min_length=1, max_length=100)
+    port: int = Field(ge=1, le=65535)
+
+
+class Resource(BaseModel):
+    model_config = ConfigDict(extra="forbid", strict=True, frozen=True)
+    id: str = Field(min_length=1, max_length=100, pattern=r"^[a-zA-Z0-9._:-]+$")
+    root: str = Field(min_length=1, max_length=4096)
+    endpoint: Endpoint
+    # Operator configuration contract; request fields cannot supply this policy.
+    write_staging_directory: str | None = Field(default=None, max_length=4096)
+
+
+class Configuration(BaseModel):
+    model_config = ConfigDict(extra="forbid", strict=True, frozen=True)
+    version: Literal[1]
+    resources: list[Resource] = Field(min_length=1, max_length=100)
+    receipt_archive_index: bool = False
+    legacy_write_staging_directories: list[str] = Field(
+        default_factory=list, max_length=100
+    )
+
+
+class Request(BaseModel):
+    # Unknown model fields have no authority and are never forwarded to the worker.
+    model_config = ConfigDict(extra="ignore", strict=True)
+    operation: Literal[
+        "status",
+        "claim",
+        "transfer",
+        "stop",
+        "read",
+        "list",
+        "stat",
+        "write",
+        "delete",
+        "receipt",
+        "ack",
+        "reconcile",
+        "resolve",
+    ]
+    root: str = Field(max_length=4096)
+    resource_id: str = Field(max_length=100)
+    owner_id: str = Field(min_length=1, max_length=200)
+    endpoint: Endpoint
+    client_journal: str | None = Field(default=None, max_length=4096)
+    path: str | None = Field(default=None, max_length=4096)
+    content: str | None = Field(default=None, max_length=1048576)
+    dry_run: bool = False
+    tool_call_id: str | None = Field(default=None, max_length=200)
+    operation_id: str | None = Field(default=None, max_length=36)
+    receipt_id: str | None = Field(default=None, max_length=36)
+    observation_token: str | None = Field(default=None, max_length=64)
+    expected_generation: int | None = None
+    target_owner_id: str | None = Field(default=None, max_length=200)
+    ttl_seconds: int | float | None = None
+    original_operation: Literal["write", "delete"] | None = None
+    original_owner_id: str | None = Field(default=None, max_length=200)
+    original_tool_call_id: str | None = Field(default=None, max_length=200)
+
+
+class ImmutableResourceBroker:
+    def __init__(self) -> None:
+        self._init_execution_system()
+        self._init_resource_system()
+        self._init_mount_system()
+
+    def _init_execution_system(self) -> None:
+        if not sys.flags.isolated or os.geteuid() == 0:
+            raise ValueError(
+                "Broker requires isolated Python and a non-root service user"
+            )
+        self.interpreter = Path(sys.executable).resolve()
+        self.worker = INSTALLATION / "worker.py"
+        self.configuration_path = INSTALLATION / "resources.json"
+        trusted = [
+            INSTALLATION,
+            self.worker,
+            self.configuration_path,
+            self.interpreter,
+            Path(sys.prefix),
+        ]
+        for path in trusted:
+            if not os.statvfs(path).f_flag & os.ST_RDONLY:
+                raise ValueError(
+                    "Broker requires a read-only execution image and policy"
+                )
+        self.trusted_devices = {path.stat().st_dev for path in trusted}
+        self.token = os.environ.get("ZTC_BROKER_TOKEN", "")
+        if not re.fullmatch(r"[!-~]{32,4096}", self.token):
+            raise ValueError(
+                "Broker requires a credential of 32 to 4096 ASCII characters"
+            )
+        self.root_identities: dict[str, tuple[int, int]] = {}
+
+    def _init_resource_system(self) -> None:
+        config = Configuration.model_validate_json(self.configuration_path.read_text())
+        self.receipt_archive_index = config.receipt_archive_index
+        self.resources = {resource.id: resource for resource in config.resources}
+        if len(self.resources) != len(config.resources):
+            raise ValueError("Broker resource IDs must be unique")
+        self.deletion_staging = Path("/data/.ztc-delete-quarantine")
+        self.deletion_staging.mkdir(mode=0o700, exist_ok=True)
+        private = self.deletion_staging.lstat()
+        if (
+            not stat.S_ISDIR(private.st_mode)
+            or private.st_uid != os.geteuid()
+            or private.st_mode & 0o077
+        ):
+            raise ValueError(
+                "Broker deletion staging must be private to its service UID"
+            )
+        self.deletion_staging_identity = (private.st_dev, private.st_ino)
+        for directory in [Path("/data"), self.deletion_staging]:
+            descriptor = os.open(
+                directory, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW
+            )
+            try:
+                os.fsync(descriptor)
+            finally:
+                os.close(descriptor)
+        for resource in config.resources:
+            root = Path(resource.root)
+            if (
+                not root.is_absolute()
+                or str(root.resolve()) != resource.root
+                or not root.is_relative_to("/data")
+                or root.is_relative_to(self.deletion_staging)
+            ):
+                raise ValueError(
+                    "Broker writable roots must be canonical directories beneath /data"
+                )
+            root.mkdir(mode=0o700, parents=True, exist_ok=True)
+            info = root.stat()
+            if not stat.S_ISDIR(info.st_mode) or info.st_dev in self.trusted_devices:
+                raise ValueError(
+                    "Broker data must use a separate filesystem "
+                    "from its executor and policy"
+                )
+            self.root_identities[resource.id] = (info.st_dev, info.st_ino)
+        self.write_staging_directories: dict[str, Path] = {}
+        self.trusted_staging_identities: dict[str, tuple[int, int]] = {
+            str(self.deletion_staging): self.deletion_staging_identity
+        }
+        for resource in config.resources:
+            self.write_staging_directories[resource.id] = (
+                Path(resource.write_staging_directory)
+                if resource.write_staging_directory is not None
+                else self.deletion_staging
+            )
+        for path in [
+            *self.write_staging_directories.values(),
+            *map(Path, config.legacy_write_staging_directories),
+        ]:
+            if (
+                not path.is_absolute()
+                or str(path.resolve(strict=True)) != str(path)
+                or not path.is_relative_to("/data")
+            ):
+                raise ValueError(
+                    "Write staging must be an existing canonical /data directory"
+                )
+            private = path.lstat()
+            if (
+                not stat.S_ISDIR(private.st_mode)
+                or private.st_uid != os.geteuid()
+                or private.st_mode & 0o077
+                or private.st_dev in self.trusted_devices
+            ):
+                raise ValueError(
+                    "Write staging must be private to the broker on its data filesystem"
+                )
+            self.trusted_staging_identities[str(path)] = (
+                private.st_dev,
+                private.st_ino,
+            )
+        os.makedirs("/state", mode=0o700, exist_ok=True)
+
+    def _init_mount_system(self) -> None:
+        snapshot = read_mount_snapshot()
+        entries = parse_mount_snapshot(snapshot)
+        self.mountinfo_sha256 = hashlib.sha256(snapshot).hexdigest()
+        self.filesystem_regions: dict[str, FilesystemRegion] = {}
+        self.resource_mount_ids: dict[str, int] = {}
+        for resource in self.resources.values():
+            root = Path(resource.root)
+            if any(
+                Path(entry.mount_point) != root
+                and Path(entry.mount_point).is_relative_to(root)
+                for entry in entries.values()
+            ):
+                raise ValueError("Nested mounts require proven overlap before serving")
+            descriptor = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
+            try:
+                info = os.fstat(descriptor)
+                details = Path(f"/proc/self/fdinfo/{descriptor}").read_text()
+                match = re.search(r"^mnt_id:\s*([0-9]+)$", details, re.MULTILINE)
+                if match is None or int(match[1]) not in entries:
+                    raise ValueError("Host root has no proven mount identity")
+                entry = entries[int(match[1])]
+                if (
+                    (info.st_dev, info.st_ino) != self.root_identities[resource.id]
+                    or entry.device
+                    != f"{os.major(info.st_dev)}:{os.minor(info.st_dev)}"
+                    or not root.is_relative_to(entry.mount_point)
+                ):
+                    raise ValueError("Host root does not match its mount identity")
+                path = str(Path(entry.root) / root.relative_to(entry.mount_point))
+                self.filesystem_regions[resource.id] = FilesystemRegion(
+                    device=entry.device, path=path
+                )
+                self.resource_mount_ids[resource.id] = entry.mount_id
+            finally:
+                os.close(descriptor)
+        staging_mount_ids: dict[str, int] = {}
+        for path, expected in self.trusted_staging_identities.items():
+            location = Path(path)
+            if any(
+                Path(entry.mount_point) != location
+                and Path(entry.mount_point).is_relative_to(location)
+                for entry in entries.values()
+            ):
+                raise ValueError("Nested private staging mounts are unsupported")
+            descriptor = os.open(location, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
+            try:
+                info = os.fstat(descriptor)
+                details = Path(f"/proc/self/fdinfo/{descriptor}").read_text()
+                match = re.search(r"^mnt_id:\s*([0-9]+)$", details, re.MULTILINE)
+                if match is None or int(match[1]) not in entries:
+                    raise ValueError("Write staging has no proven mount identity")
+                entry = entries[int(match[1])]
+                if (
+                    (info.st_dev, info.st_ino) != expected
+                    or entry.device
+                    != f"{os.major(info.st_dev)}:{os.minor(info.st_dev)}"
+                    or not location.is_relative_to(entry.mount_point)
+                ):
+                    raise ValueError("Write staging does not match its mount identity")
+                physical_path = str(
+                    Path(entry.root) / location.relative_to(entry.mount_point)
+                )
+                if any(
+                    region.device == entry.device
+                    and os.path.commonpath([physical_path, region.path])
+                    in [physical_path, region.path]
+                    for region in self.filesystem_regions.values()
+                ):
+                    raise ValueError(
+                        "Private staging overlaps a configured resource through its filesystem region"
+                    )
+                staging_mount_ids[path] = entry.mount_id
+            finally:
+                os.close(descriptor)
+        for resource in self.resources.values():
+            if resource.write_staging_directory is not None and (
+                staging_mount_ids[str(self.write_staging_directories[resource.id])]
+                != self.resource_mount_ids[resource.id]
+            ):
+                raise ValueError(
+                    "Configured write staging must share the exact resource mount"
+                )
+        self._validate_mount_system()
+
+    def _validate_mount_system(self) -> None:
+        if hashlib.sha256(read_mount_snapshot()).hexdigest() != self.mountinfo_sha256:
+            raise MountTopologyChangedError(
+                "Host mount topology changed; restart with reviewed policy"
+            )
+        for resource in self.resources.values():
+            if str(Path(resource.root).resolve()) != resource.root:
+                raise ValueError("Host broker resource canonical path changed")
+            info = os.stat(resource.root)
+            if (info.st_dev, info.st_ino) != self.root_identities[resource.id]:
+                raise ValueError("Host broker resource identity changed")
+        for path, expected in self.trusted_staging_identities.items():
+            private = Path(path).lstat()
+            if (
+                str(Path(path).resolve()) != path
+                or not stat.S_ISDIR(private.st_mode)
+                or (private.st_dev, private.st_ino) != expected
+                or private.st_uid != os.geteuid()
+                or private.st_mode & 0o077
+            ):
+                raise ValueError(
+                    "Host broker private staging identity or permissions changed"
+                )
+
+    def _prepare_request(self, data: dict[str, Any]) -> str:
+        request = Request.model_validate(data)
+        resource = self.resources.get(request.resource_id)
+        if (
+            resource is None
+            or request.root != resource.root
+            or request.endpoint != resource.endpoint
+        ):
+            raise ValueError("Request does not match a host-configured broker resource")
+        info = os.stat(resource.root)
+        identity = (info.st_dev, info.st_ino)
+        if identity != self.root_identities[resource.id]:
+            raise ValueError("Host broker resource identity changed")
+        self._validate_mount_system()
+        payload = request.model_dump(exclude_unset=True)
+        payload["host_receipt_archive_index"] = self.receipt_archive_index
+        payload["host_root_identity"] = list(identity)
+        payload["host_mountinfo_sha256"] = self.mountinfo_sha256
+        payload["host_filesystem_region"] = self.filesystem_regions[
+            resource.id
+        ].model_dump()
+        payload["host_resource_roots"] = {
+            item.root: list(self.root_identities[item.id])
+            for item in self.resources.values()
+        }
+        private = self.deletion_staging.lstat()
+        if (private.st_dev, private.st_ino) != self.deletion_staging_identity:
+            raise ValueError("Host broker deletion staging identity changed")
+        payload["host_delete_staging"] = str(self.deletion_staging)
+        payload["host_delete_staging_identity"] = list(self.deletion_staging_identity)
+        write_staging = str(self.write_staging_directories[resource.id])
+        payload["host_write_staging"] = write_staging
+        payload["host_write_staging_identity"] = list(
+            self.trusted_staging_identities[write_staging]
+        )
+        payload["host_write_staging_directories"] = {
+            path: list(identity)
+            for path, identity in self.trusted_staging_identities.items()
+        }
+        return json.dumps(payload)
+
+    def execute_request(self, data: dict[str, Any]) -> dict[str, Any]:
+        try:
+            payload = self._prepare_request(data)
+        except (ValueError, OSError) as error:
+            raise RequestNotDispatchedError(
+                str(error)
+                if isinstance(error, MountTopologyChangedError)
+                else "Request refused by immutable broker policy"
+            ) from error
+        # No shell, PATH lookup, account startup or writable Python search path.
+        # Neither credentials nor caller environment enter the worker process.
+        try:
+            result = subprocess.run(
+                [str(self.interpreter), "-I", "-B", str(self.worker)],
+                input=payload,
+                capture_output=True,
+                text=True,
+                cwd=str(INSTALLATION),
+                env={"HOME": "/state", "LANG": "C.UTF-8"},
+                timeout=30,
+                check=False,
+            )
+        except (OSError, ValueError):
+            return {
+                "ok": False,
+                "error": "Broker execution interrupted; outcome may be unresolved",
+            }
+        except subprocess.TimeoutExpired:
+            return {
+                "ok": False,
+                "error": "Broker execution timed out; outcome may be unresolved",
+            }
+        try:
+            if len(result.stdout.encode()) > MAXIMUM_BYTES:
+                return {
+                    "ok": False,
+                    "error": "Broker execution exceeded its response bound",
+                }
+            envelope = json.loads(result.stdout)
+        except (ValueError, RecursionError):
+            return {
+                "ok": False,
+                "error": "Broker execution did not return a complete receipt",
+            }
+        if not isinstance(envelope, dict) or not isinstance(envelope.get("ok"), bool):
+            return {"ok": False, "error": "Invalid broker execution envelope"}
+        if result.returncode != 0 and envelope["ok"]:
+            return {
+                "ok": False,
+                "error": "Broker execution failed before acknowledgement",
+            }
+        return envelope
+
+
+class BrokerServer(ThreadingHTTPServer):
+    def __init__(self, broker: ImmutableResourceBroker, port: int) -> None:
+        self.broker = broker
+        self.request_slots = threading.BoundedSemaphore(MAXIMUM_CONCURRENT_REQUESTS)
+        super().__init__(("0.0.0.0", port), BrokerHandler)
+
+    def process_request(
+        self, request: socket.socket, client_address: tuple[str, int]
+    ) -> None:
+        # Admission happens in the accept loop, before a handler or header timer
+        # exists. Waiting here would let incomplete peers block every new client.
+        if not self.request_slots.acquire(blocking=False):
+            encoded = json.dumps(
+                {"ok": False, "error": "Broker request capacity reached"}
+            ).encode()
+            response = (
+                "HTTP/1.1 503 Service Unavailable\r\n"
+                "Content-Type: application/json\r\n"
+                f"Content-Length: {len(encoded)}\r\n"
+                "Cache-Control: no-store\r\n"
+                "Connection: close\r\n\r\n"
+            ).encode() + encoded
+            try:
+                request.settimeout(1)
+                request.sendall(response)
+            except OSError:
+                L.debug("Overloaded broker peer disconnected")
+            finally:
+                self.shutdown_request(request)
+            return
+        try:
+            super().process_request(request, client_address)
+        except BaseException:
+            self.request_slots.release()
+            raise
+
+    def process_request_thread(
+        self, request: socket.socket, client_address: tuple[str, int]
+    ) -> None:
+        try:
+            super().process_request_thread(request, client_address)
+        finally:
+            self.request_slots.release()
+
+
+class BrokerHandler(BaseHTTPRequestHandler):
+    server: BrokerServer
+
+    def setup(self) -> None:
+        super().setup()
+        self.connection.settimeout(5)
+        self.header_timer: threading.Timer | None = None
+
+    def handle_one_request(self) -> None:
+        # Socket read timeouts bound silence only. An absolute deadline also
+        # bounds unfinished request lines/headers that continue receiving bytes.
+        self.header_timer = threading.Timer(5, self._expire_headers)
+        self.header_timer.daemon = True
+        self.header_timer.start()
+        try:
+            super().handle_one_request()
+        finally:
+            self.header_timer.cancel()
+
+    def parse_request(self) -> bool:
+        try:
+            return super().parse_request()
+        finally:
+            # Complete headers enter the ordinary authenticated/body path,
+            # whose read timeout and framing bounds remain independent.
+            if self.header_timer is not None:
+                self.header_timer.cancel()
+
+    def _expire_headers(self) -> None:
+        try:
+            self.connection.shutdown(socket.SHUT_RDWR)
+        except OSError:
+            pass  # The peer or request cleanup already closed this connection.
+
+    def log_message(self, format: str, *args: Any) -> None:
+        # Request paths, headers, payloads and credentials are never logged.
+        L.debug("Broker HTTP request completed")
+
+    def send_envelope(self, status: int, envelope: dict[str, Any]) -> None:
+        encoded = json.dumps(envelope).encode()
+        self.send_response(status)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(encoded)))
+        self.send_header("Cache-Control", "no-store")
+        self.end_headers()
+        self.wfile.write(encoded)
+
+    def do_POST(self) -> None:
+        credential = self.headers.get("Authorization", "").encode()
+        if not hmac.compare_digest(
+            credential, ("Bearer " + self.server.broker.token).encode()
+        ):
+            self.send_envelope(
+                401, {"ok": False, "error": "Broker authentication required"}
+            )
+            return
+        if (
+            self.path != "/v1/resources"
+            or self.headers.get("Transfer-Encoding") is not None
+        ):
+            self.send_envelope(
+                400, {"ok": False, "error": "Invalid broker route or framing"}
+            )
+            return
+        length = self.headers.get("Content-Length", "")
+        if (
+            not length.isdecimal()
+            or len(length) > 9
+            or not 0 < int(length) <= MAXIMUM_BYTES
+        ):
+            self.send_envelope(
+                413, {"ok": False, "error": "Broker request exceeds its bound"}
+            )
+            return
+        data: dict[str, Any] = {}
+        try:
+            raw = self.rfile.read(int(length))
+            if len(raw) != int(length):
+                raise ValueError("Incomplete broker request")
+            parsed = json.loads(raw)
+            if not isinstance(parsed, dict):
+                raise ValueError("Broker request must be an object")
+            data = parsed
+        except (ValueError, OSError, RecursionError):
+            # Body parsing cannot dispatch a worker or recover a trusted request.
+            self.send_envelope(
+                200,
+                {"ok": False, "error": "Request refused by immutable broker policy"},
+            )
+            return
+        try:
+            envelope = self.server.broker.execute_request(data)
+        except RequestNotDispatchedError as error:
+            # Only preparation can establish this proof. Other execution errors
+            # must not release a client's reservation after a possible effect.
+            envelope = {
+                "ok": False,
+                "error": str(error),
+                "admitted": False,
+                "operation_id": data.get("operation_id"),
+            }
+        except (ValueError, OSError, RecursionError):
+            envelope = {
+                "ok": False,
+                "error": "Broker execution failed; outcome may be unresolved",
+            }
+        self.send_envelope(200, envelope)
+
+
+def main() -> None:
+    logging.basicConfig(level=logging.INFO)
+    try:
+        broker = ImmutableResourceBroker()
+        port = int(os.environ.get("ZTC_BROKER_PORT", "8787"))
+        if not 1 <= port <= 65535:
+            raise ValueError("Invalid broker port")
+        with BrokerServer(broker, port) as server:
+            L.info("Immutable resource broker ready on port %d", port)
+            server.serve_forever()
+    except (ValueError, OSError) as error:
+        L.error("Broker startup refused: %s", error)
+        raise SystemExit(1) from error
+
+
+if __name__ == "__main__":
+    main()