@agentsdock/server 1.0.7-beta.5

package/server/execution_recovery_status.py

@@ -0,0 +1,195 @@
+"""Settle the exact admitted update after independent native recovery.
+
+This module and its dependencies use only the standard library: the durable
+recovery job runs independently of the candidate virtual environment. A terminal
+installer receipt, not a matching version string, authorizes settlement.
+"""
+from __future__ import annotations
+
+from contextlib import contextmanager
+from datetime import datetime, timezone
+import fcntl
+import hashlib
+import json
+import os
+from pathlib import Path
+import re
+import stat
+import subprocess
+import sys
+from typing import Any
+
+import execution_install as files
+from execution_preparation import _check_binding
+
+
+ACTIVE = {"starting", "checking", "downloading", "verifying", "installing", "restarting"}
+INTENT_KEYS = {"format", "root", "root_binding", "candidate_binding", "version",
+               "api_contract", "update_id", "server_identity"}
+
+
+def _read(path: Path) -> dict:
+    raw, _mode = files._read_file(path, private=True)
+    def unique(pairs):
+        value = {}
+        for key, item in pairs:
+            if key in value:
+                raise ValueError("duplicate update recovery status field")
+            value[key] = item
+        return value
+    value = json.loads(raw, object_pairs_hook=unique)
+    if not isinstance(value, dict):
+        raise ValueError("update recovery status is not an object")
+    return value
+
+
+@contextmanager
+def _status_lock(path: Path):
+    files._path(path)
+    files._owned_directory(path.parent)
+    lock = path.with_name(f".{path.name}.lock")
+    descriptor = os.open(lock, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600)
+    try:
+        info = os.fstat(descriptor)
+        if (not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid()
+                or info.st_nlink != 1 or stat.S_IMODE(info.st_mode) != 0o600):
+            raise PermissionError("update recovery status lock is unsafe")
+        fcntl.flock(descriptor, fcntl.LOCK_EX)
+        current = lock.lstat()
+        if (current.st_dev, current.st_ino) != (info.st_dev, info.st_ino):
+            raise RuntimeError("update recovery status lock changed")
+        yield
+    finally:
+        os.close(descriptor)
+
+
+def _intent_digest(intent: dict) -> str:
+    return hashlib.sha256(files._json_bytes(intent)).hexdigest()
+
+
+def capture_status_binding(root: Path, context: dict, *, supplied_update_id: str,
+                           expected_server_identity: str) -> dict | None:
+    """Bind old-runner status only through its already verified recovery intent."""
+    root = files._path(root)
+    state = Path(context["execution"]["runtime_dir"]).parent
+    path = files._path(state / "admin/server-update.json")
+    if not path.exists() and not path.is_symlink():
+        if supplied_update_id:
+            raise RuntimeError("admitted update status is missing")
+        return None
+    with _status_lock(path):
+        status = _read(path)
+        intent = status.get("_activation_recovery")
+        if not supplied_update_id and status.get("phase") not in ACTIVE:
+            return None  # historical receipt during an explicit manual install
+        if intent is None and not supplied_update_id:
+            raise RuntimeError("active update has no bound native recovery intent")
+        if (not isinstance(intent, dict) or set(intent) != INTENT_KEYS
+                or type(intent["format"]) is not int or intent["format"] != 1
+                or intent["root"] != str(root)
+                or intent["server_identity"] != expected_server_identity
+                or not expected_server_identity
+                or re.fullmatch(r"[0-9a-f]{32}", str(intent["update_id"])) is None
+                or intent["update_id"] != status.get("update_id")
+                or (supplied_update_id and supplied_update_id != intent["update_id"])
+                or status.get("phase") not in ACTIVE
+                or intent["version"] != status.get("target_version")
+                or intent["version"] != context["release_version"]
+                or type(intent["api_contract"]) is not int or intent["api_contract"] < 1
+                or intent["api_contract"] != context["execution"]["api_contract"]):
+            raise RuntimeError("native recovery does not own the admitted update")
+        _check_binding(root, intent["root_binding"])
+        saved, candidate = intent["candidate_binding"], context["candidate_release"]
+        if (not isinstance(saved, dict) or set(saved) != {"device", "inode", "volume_uuid"}
+                or any(type(saved[name]) is not int or saved[name] < 1 for name in ("device", "inode"))
+                or saved["inode"] != candidate["inode"]):
+            raise RuntimeError("native recovery candidate differs from the admitted update")
+        if sys.platform == "darwin":
+            if not saved["volume_uuid"] or saved["volume_uuid"] != intent["root_binding"]["volume_uuid"]:
+                raise RuntimeError("native recovery candidate volume changed")
+        elif saved["volume_uuid"] is not None or saved["device"] != candidate["device"]:
+            raise RuntimeError("native recovery candidate device changed")
+        if context["execution"]["handoff"] is not None and context["execution"]["handoff"] != status.get("_execution_handoff"):
+            raise RuntimeError("native recovery handoff differs from the admitted update")
+        return {"format": 1, "path": str(path), "update_id": intent["update_id"],
+                "target_version": intent["version"], "intent_sha256": _intent_digest(intent)}
+
+
+def _live_runner(status: dict, update_id: str) -> bool:
+    pid = status.get("runner_pid")
+    if pid is None:
+        # A newly launched recovery runner may not have published its PID yet.
+        try:
+            updated = datetime.fromisoformat(str(status["updated_at"]).replace("Z", "+00:00"))
+            return 0 <= (datetime.now(timezone.utc) - updated).total_seconds() < 45
+        except (KeyError, ValueError, TypeError):
+            return False
+    if type(pid) is not int or pid <= 1:
+        raise RuntimeError("saved update runner identity is invalid")
+    result = subprocess.run(["/bin/ps", "-ww", "-p", str(pid), "-o", "command="],
+                            capture_output=True, text=True, timeout=5, check=False)
+    if result.returncode not in {0, 1}:
+        raise RuntimeError("cannot inspect the admitted update runner")
+    command = result.stdout.strip()
+    return bool("update_runner.py" in command and re.search(
+        rf"(?:^|\s)--update-id\s+{re.escape(update_id)}(?:\s|$)", command))
+
+
+def settle_status(owner: dict, terminal: dict) -> bool:
+    """Return False while a live runner owns settlement; never replace a new row.
+
+    The caller has validated native terminal health and finalization under the
+    installation lock. This adds status ownership, not another recovery path.
+    """
+    binding = owner.get("status_binding")
+    if binding is None:
+        return True
+    if (not isinstance(binding, dict)
+            or set(binding) != {"format", "path", "update_id", "target_version", "intent_sha256"}
+            or type(binding["format"]) is not int or binding["format"] != 1
+            or binding["target_version"] != owner["version"]
+            or re.fullmatch(r"[0-9a-f]{32}", str(binding["update_id"])) is None
+            or re.fullmatch(r"[0-9a-f]{64}", str(binding["intent_sha256"])) is None):
+        raise RuntimeError("native recovery status binding is invalid")
+    path = files._path(binding["path"])
+    if path != Path(owner["state_root"]) / "admin/server-update.json":
+        raise RuntimeError("native recovery status path changed")
+    if (not isinstance(terminal, dict) or type(terminal.get("format")) is not int
+            or terminal["format"] != 1
+            or terminal.get("transaction_id") != owner["transaction_id"]
+            or not isinstance(terminal.get("snapshot"), dict)
+            or terminal.get("phase") not in {"committed", "rollback-healthy"}):
+        raise RuntimeError("native recovery terminal proof is invalid")
+    health = terminal["snapshot"].get("health")
+    if not isinstance(health, dict) or health.get("server_identity") != owner["expected_server_identity"]:
+        raise RuntimeError("native recovery terminal identity changed")
+    completed = terminal["phase"] == "committed"
+    if completed and (health.get("server_version") != owner["version"]
+            or health.get("api_contract_version") != owner["api_contract"]
+            or health.get("worker_version") != owner["version"]
+            or health.get("gateway_version") != owner["version"]):
+        raise RuntimeError("native recovery completion is not the paired target")
+    with _status_lock(path):
+        current = _read(path)
+        if current.get("update_id") != binding["update_id"] or current.get("phase") not in ACTIVE:
+            return True  # another update, or the original runner already settled
+        if (current.get("target_version") != binding["target_version"]
+                or _intent_digest(current.get("_activation_recovery")) != binding["intent_sha256"]):
+            raise RuntimeError("admitted update changed before recovery settlement")
+        if any((Path(owner["root"]) / name).exists() or (Path(owner["root"]) / name).is_symlink()
+               for name in (".activation-transaction", ".execution-transaction", ".execution-uninstall.json")):
+            raise RuntimeError("lifecycle transaction still owns recovery settlement")
+        if _live_runner(current, binding["update_id"]):
+            return False
+        now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
+        current.update(phase="complete" if completed else "failed", runner_pid=None,
+            heartbeat_at=None, elapsed_seconds=None, updated_at=now, finished_at=now,
+            error_code=None if completed else "server_update_rolled_back",
+            error_action=None if completed else "Retry the bundled update when ready.",
+            retryable=None if completed else True,
+            message=(f"AgentsServer {owner['version']} recovered and is healthy." if completed else
+                     "The interrupted update was rolled back to the verified previous installation."))
+        if completed:
+            current.update(installed_version=owner["version"], update_available=False)
+        files._atomic_write(path, files._json_bytes(current))
+        return True

package/server/execution_service.py

@@ -0,0 +1,387 @@
+"""Independent execution owner and replaceable public ASGI gateway.
+
+The worker owns the application's lifespan, provider transports and state. The
+gateway owns only public client connections. In particular, losing the gateway
+must never run the application's shutdown handlers or stop its provider children.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import errno
+import fcntl
+import importlib
+import json
+import logging
+import os
+from pathlib import Path
+import socket
+import stat
+import time
+from typing import Any
+import uuid
+
+import uvicorn
+
+from execution_transport import (
+    ExecutionGateway,
+    ExecutionTransportServer,
+    ensure_execution_secret,
+)
+
+
+PROTOCOL_VERSION = 1
+MAX_HEALTH_BYTES = 1024 * 1024
+
+
+class ProcessLease:
+    """One owner per role, independent of PIDs reused after a crash."""
+
+    def __init__(self, runtime_dir: Path, role: str) -> None:
+        if role not in {"worker", "gateway"}:
+            raise ValueError("Unknown execution service role")
+        self.directory = runtime_dir
+        self.role = role
+        self.fd: int | None = None
+        self.instance_id = uuid.uuid4().hex
+
+    def __enter__(self) -> "ProcessLease":
+        self.directory.mkdir(mode=0o700, parents=True, exist_ok=True)
+        ensure_execution_secret(self.directory / "control.token")
+        path = self.directory / f"{self.role}.lock"
+        fd = os.open(path, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600)
+        try:
+            info = os.fstat(fd)
+            if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid():
+                raise ValueError("Execution lease is not an owned regular file")
+            if stat.S_IMODE(info.st_mode) != 0o600:
+                raise ValueError("Execution lease must be owner-only")
+            fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            os.set_inheritable(fd, False)
+        except BaseException:
+            os.close(fd)
+            raise
+        self.fd = fd
+        return self
+
+    def publish(self, **values: Any) -> None:
+        if self.fd is None:
+            raise RuntimeError("Execution lease is not held")
+        receipt = {
+            "schema": 1,
+            "protocol": PROTOCOL_VERSION,
+            "role": self.role,
+            "pid": os.getpid(),
+            "instance_id": self.instance_id,
+            "started_at": time.time(),
+            **values,
+        }
+        temporary = self.directory / f".{self.role}.{self.instance_id}.tmp"
+        fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
+        try:
+            with os.fdopen(fd, "w", encoding="utf-8") as stream:
+                json.dump(receipt, stream, sort_keys=True)
+                stream.write("\n")
+                stream.flush()
+                os.fsync(stream.fileno())
+            os.replace(temporary, self.directory / f"{self.role}.json")
+        finally:
+            temporary.unlink(missing_ok=True)
+
+    def __exit__(self, *unused: Any) -> None:
+        # The lock remains held while removing this process's receipt. Leave the
+        # lock inode in place so a waiting process cannot acquire another inode.
+        receipt_path = self.directory / f"{self.role}.json"
+        try:
+            if not receipt_path.is_symlink():
+                receipt = json.loads(receipt_path.read_text())
+                if isinstance(receipt, dict) and receipt.get("instance_id") == self.instance_id:
+                    receipt_path.unlink()
+        except (FileNotFoundError, ValueError, OSError):
+            pass
+        finally:
+            if self.fd is not None:
+                os.close(self.fd)
+                self.fd = None
+
+
+class ComponentHealth:
+    """Add component identity without relabeling an old engine as upgraded."""
+
+    def __init__(self, app: Any, field: str, identity: dict[str, Any], live_identity: Any = None) -> None:
+        self.app = app
+        self.field = field
+        self.identity = dict(identity)
+        self.live_identity = live_identity
+
+    async def __call__(self, scope: dict[str, Any], receive: Any, send: Any) -> None:
+        if scope["type"] != "http" or scope.get("path") != "/api/health":
+            await self.app(scope, receive, send)
+            return
+        start: dict[str, Any] | None = None
+        chunks: list[bytes] = []
+        count = 0
+        passthrough = False
+
+        async def component_send(message: dict[str, Any]) -> None:
+            nonlocal start, count, passthrough
+            if message["type"] == "http.response.start":
+                headers = message.get("headers", [])
+                content_types = [v for k, v in headers if k.lower() == b"content-type"]
+                encoded = any(k.lower() == b"content-encoding" for k, _ in headers)
+                if message["status"] != 200 or encoded or not any(
+                    v.lower().startswith(b"application/json") for v in content_types
+                ):
+                    passthrough = True
+                    await send(message)
+                else:
+                    start = dict(message)
+                return
+            if passthrough or start is None or message["type"] != "http.response.body":
+                await send(message)
+                return
+            chunk = message.get("body", b"")
+            chunks.append(chunk)
+            count += len(chunk)
+            if count > MAX_HEALTH_BYTES:
+                passthrough = True
+                await send(start)
+                await send({**message, "body": b"".join(chunks)})
+                chunks.clear()
+                return
+            if message.get("more_body", False):
+                return
+            body = b"".join(chunks)
+            try:
+                document = json.loads(body)
+                if isinstance(document, dict):
+                    document[self.field] = {**self.identity,
+                        **(self.live_identity() if self.live_identity is not None else {})}
+                    body = json.dumps(document, separators=(",", ":")).encode()
+            except (ValueError, UnicodeError):
+                pass
+            headers = [(k, v) for k, v in start.get("headers", []) if k.lower() != b"content-length"]
+            headers.append((b"content-length", str(len(body)).encode("ascii")))
+            await send({**start, "headers": headers})
+            await send({**message, "body": body})
+
+        await self.app(scope, receive, component_send)
+
+
+class WorkerLifespan:
+    """Start IPC after the application's one startup and stop it before teardown."""
+
+    def __init__(self, app: Any, transport: ExecutionTransportServer, ready: Any) -> None:
+        self.app = app
+        self.transport = transport
+        self.ready = ready
+        self.started = False
+
+    async def __call__(self, scope: dict[str, Any], receive: Any, send: Any) -> None:
+        if scope["type"] != "lifespan":
+            await self.app(scope, receive, send)
+            return
+
+        async def lifecycle_receive() -> dict[str, Any]:
+            message = await receive()
+            if message["type"] == "lifespan.shutdown" and self.started:
+                await self.transport.close()
+                self.started = False
+            return message
+
+        async def lifecycle_send(message: dict[str, Any]) -> None:
+            if message["type"] == "lifespan.startup.complete":
+                await self.transport.start()
+                self.started = True
+                self.ready()
+            await send(message)
+
+        try:
+            await self.app(scope, lifecycle_receive, lifecycle_send)
+        finally:
+            if self.started:
+                await self.transport.close()
+                self.started = False
+
+
+def load_application(reference: str) -> tuple[Any, Any]:
+    module_name, separator, attribute = reference.partition(":")
+    if not separator or not module_name or not attribute or ":" in attribute:
+        raise ValueError("Application must be an importable module:attribute")
+    module = importlib.import_module(module_name)
+    return module, getattr(module, attribute)
+
+
+def source_version() -> str:
+    return Path(__file__).with_name("VERSION").read_text().strip()
+
+
+def remove_stale_worker_socket(path: Path) -> None:
+    """Recover a crashed listener only while the caller holds the worker lease."""
+
+    try:
+        before = path.lstat()
+    except FileNotFoundError:
+        return
+    if not stat.S_ISSOCK(before.st_mode) or before.st_uid != os.getuid():
+        raise ValueError("Refusing to replace an unexpected execution socket")
+    probe = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    probe.settimeout(0.5)
+    try:
+        probe.connect(str(path))
+    except OSError as error:
+        if error.errno != errno.ECONNREFUSED:
+            raise RuntimeError("Execution listener ownership is ambiguous") from error
+    else:
+        raise RuntimeError("An execution listener is still running")
+    finally:
+        probe.close()
+    after = path.lstat()
+    if (before.st_dev, before.st_ino) != (after.st_dev, after.st_ino):
+        raise RuntimeError("Execution socket changed during recovery")
+    path.unlink()
+
+
+async def serve_worker(args: argparse.Namespace, lease: ProcessLease) -> None:
+    module, app = load_application(args.application)
+    if args.application == "agent_server:app":
+        expected_runtime = module.STATE_DIR.expanduser().resolve() / "execution"
+        if args.runtime_dir != expected_runtime:
+            raise ValueError("Worker runtime directory must be its state directory's execution folder")
+    remove_stale_worker_socket(args.runtime_dir / "worker.socket")
+    callback_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    callback_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    callback_socket.bind(("127.0.0.1", args.callback_port))
+    callback_socket.listen(128)
+    callback_socket.setblocking(False)
+    callback_port = callback_socket.getsockname()[1]
+    callback_origin = f"http://127.0.0.1:{callback_port}"
+    maintenance = None
+    if args.application == "agent_server:app":
+        # Public addresses still describe the gateway, including Team Hub URLs.
+        # Only provider callbacks use the worker-owned stable private listener.
+        module.SERVER_BIND_ADDRESS = args.bind
+        module.SERVER_PORT = args.port
+        module.PROVIDER_CALLBACK_ORIGIN = callback_origin
+        module.configure_server_logging(module.STATE_DIR)
+        from execution_install import pending_worker_operation
+        from execution_maintenance import ExecutionMaintenance
+
+        maintenance = ExecutionMaintenance(
+            module.SERVER_ADMIN_ROOT / "execution-maintenance.json", lease.instance_id,
+        )
+        configured_install = os.environ.get("AGENTS_SERVER_INSTALL_DIR")
+        if configured_install:
+            operation = pending_worker_operation(
+                Path(configured_install).expanduser().resolve(),
+                Path(__file__).resolve().parent,
+            )
+            if operation is not None:
+                maintenance.hold_for_startup(operation)
+        # Set before app startup so recovered queues and scheduled work cannot
+        # start in a candidate that has not passed activation health checks.
+        module.EXECUTION_MAINTENANCE = maintenance
+    version = str(getattr(module, "SERVER_VERSION", source_version()))
+    identity = {
+        "protocol": PROTOCOL_VERSION,
+        "instance_id": lease.instance_id,
+        "pid": os.getpid(),
+        "version": version,
+        "worker_upgrade_policy": "when_idle",
+        "rolling_worker_upgrade": False,
+    }
+    health_app = ComponentHealth(app, "execution_service", identity,
+        (lambda: {"maintenance_held": maintenance.is_held()}) if maintenance is not None else None)
+    transport = ExecutionTransportServer(
+        health_app,
+        socket_path=args.runtime_dir / "worker.socket",
+        secret_path=args.runtime_dir / "control.token",
+    )
+    lifecycle_app = WorkerLifespan(
+        health_app,
+        transport,
+        lambda: lease.publish(
+            version=version,
+            release_root=str(Path(__file__).resolve().parent),
+            callback_origin=callback_origin,
+            public_bind=args.bind,
+            public_port=args.port,
+        ),
+    )
+    callback_app = lifecycle_app
+    if maintenance is not None:
+        from execution_control import ExecutionControl
+
+        callback_app = ExecutionControl(
+            lifecycle_app,
+            token=ensure_execution_secret(args.runtime_dir / "control.token"),
+            worker_instance_id=lease.instance_id,
+            status_callback=module.execution_maintenance_control,
+            maintenance_callback=module.execution_maintenance_control,
+        )
+    server = uvicorn.Server(uvicorn.Config(
+        callback_app, host="127.0.0.1", port=callback_port,
+        lifespan="on", proxy_headers=False, server_header=False,
+        log_level="info", timeout_graceful_shutdown=20,
+        **({"log_config": None} if args.application == "agent_server:app" else {}),
+    ))
+    try:
+        await server.serve(sockets=[callback_socket])
+    finally:
+        callback_socket.close()
+
+
+async def serve_gateway(args: argparse.Namespace, lease: ProcessLease) -> None:
+    gateway = ExecutionGateway(
+        socket_path=args.runtime_dir / "worker.socket",
+        secret_path=args.runtime_dir / "control.token",
+    )
+    identity = {
+        "protocol": PROTOCOL_VERSION,
+        "instance_id": lease.instance_id,
+        "pid": os.getpid(),
+        "version": source_version(),
+        "restart_preserves_execution": True,
+    }
+    app = ComponentHealth(gateway, "gateway", identity)
+    lease.publish(
+        version=source_version(), release_root=str(Path(__file__).resolve().parent),
+        public_bind=args.bind, public_port=args.port,
+    )
+    server = uvicorn.Server(uvicorn.Config(
+        app, host=args.bind, port=args.port,
+        lifespan="off", proxy_headers=False, server_header=False,
+        # Only client attachments belong to this process. There is no reason
+        # to delay replacement while an execution-owned request keeps running.
+        timeout_graceful_shutdown=2, log_level="warning", access_log=False,
+    ))
+    await server.serve()
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="AgentsServer execution service")
+    parser.add_argument("role", choices=["worker", "gateway"])
+    parser.add_argument("--runtime-dir", type=Path, required=True)
+    parser.add_argument("--bind", default="127.0.0.1")
+    parser.add_argument("--port", type=int, default=7850)
+    parser.add_argument("--callback-port", type=int, default=0)
+    parser.add_argument("--application", default="agent_server:app")
+    args = parser.parse_args()
+    if not args.runtime_dir.is_absolute():
+        parser.error("--runtime-dir must be absolute")
+    if not 0 <= args.callback_port <= 65535 or not 1 <= args.port <= 65535:
+        parser.error("Invalid listener port")
+    if args.role == "gateway" and args.application != "agent_server:app":
+        parser.error("Only workers load applications")
+    try:
+        with ProcessLease(args.runtime_dir, args.role) as lease:
+            asyncio.run(serve_worker(args, lease) if args.role == "worker" else serve_gateway(args, lease))
+    except BlockingIOError:
+        logging.error("Another %s already owns this execution directory", args.role)
+        return 2
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())