@crvouga/dev-fleet 0.0.0-stage → 0.2.0

package/cliproxy/scripts/checkpoint.py

@@ -0,0 +1,226 @@
+"""Consistent, secret-minimal checkpoints. Runtime state is never executable code."""
+
+import fnmatch
+import hashlib
+import json
+import os
+import pathlib
+import shutil
+import stat
+import subprocess
+import time
+
+STATE_PATHS = (
+    "runtime/config.yaml",
+    "runtime/auths",
+    "runtime/static/management.html",
+    "tailscale",
+    "endpoint.json",
+    "public-endpoint-enabled",
+    "repository.json",
+    "status/gc-success.json",
+)
+
+
+def regular_tree(path):
+    """Reject links/devices before copying or publishing recovered state."""
+    mode = path.lstat().st_mode
+    if stat.S_ISDIR(mode):
+        for child in path.iterdir():
+            regular_tree(child)
+    elif not stat.S_ISREG(mode):
+        raise ValueError("Checkpoint must contain only regular files/directories")
+
+
+def hashes(root):
+    regular_tree(root)
+    return {
+        str(path.relative_to(root)): hashlib.sha256(path.read_bytes()).hexdigest()
+        for path in sorted(root.rglob("*"))
+        if path.is_file() and path != root / "CHECKPOINT.json"
+    }
+
+
+# Upstream writes these beside credentials; they are never checkpointed.
+AUTH_TRANSIENT = ("*.lock", "logs")
+
+
+def validate(root):
+    regular_tree(root)
+    manifest = json.loads((root / "CHECKPOINT.json").read_text())
+    if manifest.get("version") != 1 or manifest.get("files") != hashes(root):
+        raise ValueError("Checkpoint incomplete or changed")
+    state = root / "state"
+    for path in root.rglob("*"):
+        if not path.is_file():
+            continue
+        name = str(path.relative_to(root))
+        if name in ("CHECKPOINT.json", "INSTALLATION.json"):
+            continue
+        if not any(
+            name == "state/" + allowed or name.startswith("state/" + allowed + "/")
+            for allowed in STATE_PATHS
+        ):
+            raise ValueError("Unexpected checkpoint file")
+    if not (state / "runtime/static/management.html").is_file():
+        raise ValueError("Dashboard missing")
+    if not (state / "runtime/auths").is_dir():
+        raise ValueError("Provider cache directory missing")
+    for path in (state / "runtime/auths").iterdir():
+        if path.name == "kv.db" and path.is_file() and path.stat().st_size > 0:
+            continue  # Opaque upstream bbolt state, captured while the process is stopped.
+        # .json: provider credential; .cds: upstream per-account cooldown
+        # records (JSON object, no credential), kept so limits survive moves.
+        if path.suffix not in (".json", ".cds") or not path.is_file():
+            raise ValueError("Unexpected provider file")
+        if not isinstance(json.loads(path.read_text()), dict):
+            raise ValueError("Provider credential must be an object")
+    subprocess.run(
+        [
+            str(pathlib.Path(__file__).parent / "validate-config"),
+            str(state / "runtime/config.yaml"),
+        ],
+        check=True,
+        capture_output=True,
+    )
+    return manifest
+
+
+def stamps(root):
+    """Size and modification time of every state file whose copy must be whole:
+    the runtime tree and the Tailscale identity. Changing logs are not state."""
+    found = {}
+    for name in ("runtime", "tailscale/tailscaled.state"):
+        source = root / "state" / name
+        paths = [source] if source.is_file() else source.rglob("*") if source.is_dir() else []
+        for path in paths:
+            if path.is_file() and not any(
+                fnmatch.fnmatch(part, pattern)
+                for part in path.relative_to(root).parts
+                for pattern in AUTH_TRANSIENT
+            ):
+                status = path.stat()
+                found[str(path)] = (status.st_size, status.st_mtime_ns)
+    return found
+
+
+def copy_state(root, target):
+    target.mkdir(mode=0o700)
+    (target / "state").mkdir(mode=0o700)
+    for name in STATE_PATHS:
+        source = root / "state" / name
+        if not source.exists() and not source.is_symlink():
+            continue
+        regular_tree(source)
+        destination = target / "state" / name
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        if source.is_dir():
+            # Process-local locks and upstream error logs (request
+            # content) are never restored.
+            ignore = (
+                shutil.ignore_patterns(*AUTH_TRANSIENT)
+                if name == "runtime/auths"
+                else None
+            )
+            shutil.copytree(source, destination, ignore=ignore)
+        else:
+            shutil.copyfile(source, destination)
+    shutil.copyfile(root / "INSTALLATION.json", target / "INSTALLATION.json")
+    (target / "CHECKPOINT.json").write_text(
+        json.dumps({"version": 1, "files": hashes(target)}) + "\n"
+    )
+    validate(target)
+
+
+def capture(root, target, attempts=6, sleep=time.sleep):
+    """Copies the state of the running stack without ever stopping it: agents
+    use the proxy around the clock, and a stopped connector fails every one of
+    their requests. A copy is consistent when no state file changed while it
+    was taken (the proxy writes a file, including its bbolt store, in one
+    synced step); otherwise it is discarded and taken again. A stack that
+    never holds still fails the backup, which the monitor reports."""
+    for attempt in range(attempts):
+        before = stamps(root)
+        try:
+            copy_state(root, target)
+            if stamps(root) == before:
+                return
+        except (OSError, ValueError):
+            # A file replaced mid-copy; only the last attempt's error counts.
+            if attempt + 1 == attempts:
+                shutil.rmtree(target, ignore_errors=True)
+                raise
+        shutil.rmtree(target, ignore_errors=True)
+        sleep(2)
+    raise ValueError("State kept changing during capture")
+
+
+def locate(restored):
+    """Restic retains the source absolute path; never assume a source hostname/path."""
+    candidates = list(restored.rglob("CHECKPOINT.json"))
+    if len(candidates) != 1:
+        raise ValueError(
+            "Expected exactly one modern checkpoint; use manual legacy recovery"
+        )
+    root = candidates[0].parent
+    validate(root)
+    return root
+
+
+def hydrate(checkpoint, state):
+    """Populate an empty cache only. A crash marker prevents partial-cache startup."""
+    validate(checkpoint)
+    names = (
+        "runtime",
+        "tailscale",
+        "endpoint.json",
+        "public-endpoint-enabled",
+        "repository.json",
+    )
+    repository = state / "repository.json"
+    if repository.exists() or repository.is_symlink():
+        regular_tree(repository)
+        saved = checkpoint / "state/repository.json"
+        if not saved.is_file() or json.loads(repository.read_text()) != json.loads(
+            saved.read_text()
+        ):
+            raise ValueError("Existing repository identity differs from checkpoint")
+        names = tuple(name for name in names if name != "repository.json")
+    if any((state / name).exists() or (state / name).is_symlink() for name in names):
+        raise ValueError(
+            "Recovery requires an empty cache; preserve or remove existing cache explicitly"
+        )
+    marker = state / "recovery-incomplete"
+    if marker.exists():
+        raise ValueError("Interrupted recovery requires inspection")
+    marker.write_text("Recovery in progress; do not start services.\n")
+    created = []
+    try:
+        for name in names:
+            source = checkpoint / "state" / name
+            if not source.exists():
+                continue
+            destination = state / name
+            created.append(destination)
+            if source.is_dir():
+                shutil.copytree(source, destination)
+            else:
+                shutil.copyfile(source, destination)
+            paths = (
+                [destination, *destination.rglob("*")]
+                if destination.is_dir()
+                else [destination]
+            )
+            for path in paths:
+                path.chmod(0o700 if path.is_dir() else 0o600)
+                if os.geteuid() == 0:
+                    os.chown(path, 1000, 1000)
+        marker.unlink()
+    except BaseException:
+        for destination in reversed(created):
+            if destination.is_dir():
+                shutil.rmtree(destination)
+            else:
+                destination.unlink(missing_ok=True)
+        marker.unlink()
+        raise

package/cliproxy/scripts/cli.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+"""Repository entry point for installing and operating the local service."""
+
+import sys
+
+sys.dont_write_bytecode = True
+
+import pathlib as _pathlib
+
+sys.path.insert(0, str(_pathlib.Path(__file__).resolve().parent))
+import cliproxy_host
+
+
+import os
+import pathlib
+import subprocess
+import sys
+
+source = pathlib.Path(__file__).resolve().parent
+root = cliproxy_host.default_root().resolve()
+args = sys.argv[1:]
+if not args or args[0] in ["help", "--help", "-h"]:
+    print(
+        "Usage: devfleet cliproxy COMMAND\nCommands: install [--revision REV|--rollback|--dry-run|--prepare-only], audit, public-endpoint [enable|start|disable|verify], status, start, stop, restart, health, models, logs, validate, apply FILE, add-account PROVIDER, inference PROTOCOL MODEL, model-health [--profile liveness|readiness|light|nightly|manual] [--suite core|cursor|all] [--target live|mock], model-suite (same flags; defaults to nightly), backup, gc, snapshots, check, check-full, restore SNAPSHOT EMPTY_DIRECTORY, restore-cache --previous-node-stopped, vault-login, recover, recover-node NEW_DIRECTORY [--vault-token-file FILE], verify-recovery, portable-login, activate, drain"
+    )
+    sys.exit(0)
+command, *rest = args
+if command == "install":
+    invocation = [sys.executable, str(source / "install.py"), *rest]
+    # The legacy Linux host installs system units; the macOS manager is per-user.
+    linux = cliproxy_host.system() == "linux"
+    if linux and os.geteuid() != 0 and "--dry-run" not in rest:
+        invocation = ["sudo", *invocation]
+elif command in ["model-health", "model-suite"]:
+    invocation = [sys.executable, "-B", str(source / "model_health.py"), command, *rest]
+elif command == "recover-node":
+    invocation = [sys.executable, "-B", str(root / "scripts/recover_node.py"), *rest]
+elif command == "public-endpoint":
+    invocation = [str(root / "scripts/public-endpoint"), *rest]
+elif command == "audit":
+    invocation = [sys.executable, str(root / "scripts/audit.py"), *rest]
+elif command in [
+    "backup",
+    "gc",
+    "snapshots",
+    "check",
+    "check-full",
+    "restore",
+    "restore-cache",
+]:
+    invocation = [str(root / "scripts/backup"), command, *rest]
+elif command in ["vault-login", "recover", "verify-recovery"]:
+    invocation = [
+        str(
+            root
+            / "scripts"
+            / ("vault-backup-login" if command == "vault-login" else "vault-recovery")
+        )
+    ]
+    if command != "vault-login":
+        invocation.append("recover" if command == "recover" else "verify")
+else:
+    invocation = [str(root / "scripts/ops"), command, *rest]
+raise SystemExit(subprocess.call(invocation))

package/cliproxy/scripts/cliproxy_host.py

@@ -0,0 +1,257 @@
+"""Explicit host dispatch for the singleton proxy.
+
+The manager (macOS) hosts the proxy with per-user launchd agents and Docker
+Desktop; no sudo. Linux systemd hosting is legacy and kept for fencing and
+decommissioning the previous owner. Everything here is pure or takes injected
+commands so it can be tested offline.
+"""
+
+import sys
+
+sys.dont_write_bytecode = True
+
+import os
+import pathlib
+import re
+import socket
+import subprocess
+
+LINUX_ROOT = pathlib.Path("/opt/cliproxy-fleet")
+# The tracked LaunchAgents reference this path relative to $HOME.
+MACOS_ROOT = "Library/Application Support/cliproxy-fleet"
+LINUX_IDS = (1000, 1000)
+# Base PATH of the tracked LaunchAgents. Each agent puts the install's
+# tools/bin (pinned python3 + PyYAML venv, pinned restic) in front of it.
+LAUNCHD_PATH = "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Applications/Docker.app/Contents/Resources/bin"
+LAUNCHD_TOOLS = "$HOME/Library/Application Support/cliproxy-fleet/tools/bin"
+DOCKER_DESKTOP_BIN = "/Applications/Docker.app/Contents/Resources/bin"
+DOCKER_UNAVAILABLE = 69  # EX_UNAVAILABLE
+
+SYSTEMD_SERVICE = "cliproxy-fleet.service"
+SYSTEMD_SCHEDULED = [
+    "cliproxy-monitor.timer",
+    "cliproxy-vault-renew.timer",
+    "cliproxy-backup.timer",
+    "cliproxy-backup-check.timer",
+    "cliproxy-gc.timer",
+]
+LAUNCHD_SERVICE = "dev.devfleet.cliproxy-fleet"
+LAUNCHD_SCHEDULED = [
+    "dev.devfleet.cliproxy-monitor",
+    "dev.devfleet.cliproxy-vault-renew",
+    "dev.devfleet.cliproxy-backup",
+    "dev.devfleet.cliproxy-backup-check",
+    "dev.devfleet.cliproxy-gc",
+]
+
+
+def system(platform=None):
+    platform = platform or sys.platform
+    if platform == "darwin":
+        return "macos"
+    if platform.startswith("linux"):
+        return "linux"
+    raise ValueError("Unsupported proxy host platform")
+
+
+def default_root(environ=None, platform=None, home=None):
+    """CLIPROXY_ROOT wins; otherwise the platform's installation root."""
+    environ = os.environ if environ is None else environ
+    if environ.get("CLIPROXY_ROOT"):
+        return pathlib.Path(environ["CLIPROXY_ROOT"])
+    if system(platform) == "macos":
+        return pathlib.Path(home or pathlib.Path.home()) / MACOS_ROOT
+    return LINUX_ROOT
+
+
+def runtime_ids(platform=None):
+    """Container/state owner: the installing user on macOS, UID/GID 1000 on Linux."""
+    if system(platform) == "macos":
+        return os.getuid(), os.getgid()
+    return LINUX_IDS
+
+
+def runtime_home(platform=None):
+    if system(platform) == "macos":
+        return pathlib.Path.home()
+    import pwd
+
+    return pathlib.Path(pwd.getpwuid(LINUX_IDS[0]).pw_dir)
+
+
+def normalize_node(name):
+    name = name.strip().lower()
+    name = name.removesuffix(".local")
+    return re.sub(r"[^a-z0-9-]", "-", name)
+
+
+def local_node_id(platform=None, output=subprocess.check_output):
+    """macOS LocalHostName (scutil) or the Linux hostname, normalized."""
+    if system(platform) == "macos":
+        name = output(["scutil", "--get", "LocalHostName"], text=True, timeout=10)
+    else:
+        name = socket.gethostname()
+    return normalize_node(name)
+
+
+def _executable(path):
+    return os.path.isfile(path) and os.access(path, os.X_OK)
+
+
+def _reachable(path):
+    connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    connection.settimeout(2)
+    try:
+        connection.connect(path)
+        return True
+    except OSError:
+        return False
+    finally:
+        connection.close()
+
+
+class DockerUnavailable(Exception):
+    def __init__(self, code, message):
+        super().__init__(message)
+        self.code = code
+
+
+def docker_invocation(
+    args,
+    wrapper,
+    platform=None,
+    environ=None,
+    home=None,
+    executable=_executable,
+    reachable=_reachable,
+):
+    """Return (argv, environment) for the real Docker CLI behind the tracked wrapper."""
+    environ = dict(os.environ if environ is None else environ)
+    if system(platform) == "linux":
+        if os.access("/var/run/docker.sock", os.R_OK | os.W_OK):
+            return ["/usr/bin/docker", *args], environ
+        return ["sudo", "-n", "/usr/bin/docker", *args], environ
+    home = pathlib.Path(home or pathlib.Path.home())
+    own = os.path.realpath(wrapper)
+    candidates = [
+        os.path.join(directory, "docker")
+        for directory in environ.get("PATH", "").split(os.pathsep)
+        if directory
+    ] + [
+        str(home / ".docker/bin/docker"),
+        "/usr/local/bin/docker",
+        "/opt/homebrew/bin/docker",
+        DOCKER_DESKTOP_BIN + "/docker",
+        str(home / ".local/share/mise/shims/docker"),
+    ]
+    binary = next(
+        (
+            path
+            for path in candidates
+            if executable(path) and os.path.realpath(path) != own
+        ),
+        None,
+    )
+    if binary is None:
+        raise DockerUnavailable(
+            "DOCKER_CLI_MISSING",
+            "DOCKER_CLI_MISSING: install Docker Desktop (it provides the docker CLI).",
+        )
+    host = environ.get("DOCKER_HOST", "")
+    if host and not host.startswith("unix://"):
+        socket_path = None  # Remote engines are the operator's explicit choice.
+    elif host:
+        socket_path = host.removeprefix("unix://")
+        if not reachable(socket_path):
+            socket_path = ""
+    else:
+        socket_path = next(
+            (
+                path
+                for path in [str(home / ".docker/run/docker.sock"), "/var/run/docker.sock"]
+                if reachable(path)
+            ),
+            "",
+        )
+        if socket_path and not environ.get("DOCKER_CONTEXT"):
+            environ["DOCKER_HOST"] = "unix://" + socket_path
+    if socket_path == "":
+        raise DockerUnavailable(
+            "DOCKER_DESKTOP_NOT_RUNNING",
+            "DOCKER_DESKTOP_NOT_RUNNING: start Docker Desktop and enable "
+            "'Start Docker Desktop when you sign in'; nothing was changed.",
+        )
+    # Credential helpers (docker-credential-desktop) live beside the CLI.
+    environ["PATH"] = os.pathsep.join(
+        [os.path.dirname(binary), DOCKER_DESKTOP_BIN, environ.get("PATH", "")]
+    )
+    return [binary, *args], environ
+
+
+class Systemd:
+    """Legacy Linux owner backend: fencing/decommission of the previous host."""
+
+    name = "systemd"
+    service = SYSTEMD_SERVICE
+    scheduled = SYSTEMD_SCHEDULED
+    recovery_gated = set(SYSTEMD_SCHEDULED[2:])
+    units = [SYSTEMD_SERVICE, *SYSTEMD_SCHEDULED]
+    unit_directory = pathlib.Path("/etc/systemd/system")
+    source_directory = "systemd"
+
+    def state(self, unit, run):
+        return (
+            run(["systemctl", "is-active", "--quiet", unit]),
+            run(["systemctl", "is-enabled", "--quiet", unit]),
+        )
+
+    def enable(self, unit):
+        return [["sudo", "-n", "systemctl", "enable", "--now", unit]]
+
+    def install(self, python, scripts, revision):
+        return ["sudo", "-n", python, "-B", str(scripts / "install.py"), "--revision", revision]
+
+
+class Launchd:
+    """Manager owner backend: per-user LaunchAgents in gui/$UID, no root."""
+
+    name = "launchd"
+    service = LAUNCHD_SERVICE
+    scheduled = LAUNCHD_SCHEDULED
+    recovery_gated = set(LAUNCHD_SCHEDULED[2:])
+    # Load scheduled agents first: service-start's audit expects them loaded.
+    units = [*LAUNCHD_SCHEDULED, LAUNCHD_SERVICE]
+    source_directory = "launchd"
+
+    def __init__(self, uid=None, home=None):
+        self.uid = os.getuid() if uid is None else uid
+        self.home = pathlib.Path(home or pathlib.Path.home())
+        self.domain = "gui/" + str(self.uid)
+        self.unit_directory = self.home / "Library/LaunchAgents"
+
+    def plist(self, label):
+        return self.unit_directory / (label + ".plist")
+
+    def loaded(self, label, run):
+        return run(["launchctl", "print", self.domain + "/" + label])
+
+    def state(self, unit, run):
+        loaded = self.loaded(unit, run)
+        return loaded, loaded and self.plist(unit).is_symlink()
+
+    def enable(self, unit):
+        # A prior `launchctl disable` persists; clear it before loading.
+        return [
+            ["launchctl", "enable", self.domain + "/" + unit],
+            ["launchctl", "bootstrap", self.domain, str(self.plist(unit))],
+        ]
+
+    def bootout(self, unit):
+        return ["launchctl", "bootout", self.domain + "/" + unit]
+
+    def install(self, python, scripts, revision):
+        return [python, "-B", str(scripts / "install.py"), "--revision", revision]
+
+
+def backend(platform=None):
+    return Launchd() if system(platform) == "macos" else Systemd()

package/cliproxy/scripts/cloud_state.py

@@ -0,0 +1,150 @@
+"""Read-only cloud recovery helpers; no dependency on the running workstation."""
+
+import datetime
+import json
+import os
+import pathlib
+import re
+import subprocess
+import urllib.error
+import urllib.request
+
+import sys as _sys, pathlib as _p
+_sys.path.insert(0, str(_p.Path(__file__).resolve().parent))
+import fleet_config
+
+
+def shared_url():
+    """The shared Vault document holding the proxy's keys and backup credentials."""
+    vault = fleet_config.vault()
+    return vault["address"] + "/v1/" + vault["shared"]
+
+FIELDS = (
+    "S3_ENDPOINT",
+    "S3_BUCKET",
+    "S3_REGION",
+    "S3_ACCESS_KEY_ID",
+    "S3_SECRET_ACCESS_KEY",
+    "CLIPROXY_FLEET_RESTIC_PASSWORD",
+    "CLIPROXY_FLEET_CLIENT_API_KEY",
+    "CLIPROXY_FLEET_MANAGEMENT_KEY",
+)
+
+
+class NoRedirect(urllib.request.HTTPRedirectHandler):
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        return None  # Never forward a Vault token or API key to a redirect destination.
+
+
+def request(url, headers=None):
+    if not url.startswith("https://"):
+        raise ValueError("HTTPS required")
+    headers = {"User-Agent": "cliproxy-fleet/1.0", **(headers or {})}
+    try:
+        with urllib.request.build_opener(NoRedirect).open(
+            urllib.request.Request(url, headers=headers), timeout=30
+        ) as response:
+            return response.status, response.read()
+    except urllib.error.HTTPError as error:
+        return error.code, b""
+
+
+def settings(token):
+    status, body = request(
+        shared_url(), {"X-Vault-Token": token}
+    )
+    if status != 200:
+        raise ValueError("Vault read failed")
+    data = json.loads(body)["data"]["data"]
+    if any(
+        not isinstance(data.get(name), str) or not data[name].strip() for name in FIELDS
+    ):
+        raise ValueError("Cloud recovery contract is incomplete")
+    return {name: data[name] for name in FIELDS}
+
+
+def restic_environment(data):
+    if not data["S3_ENDPOINT"].startswith("https://"):
+        raise ValueError("Storage endpoint must use HTTPS")
+    # Do not inherit competing Restic destinations, passwords, AWS profiles or session tokens.
+    return {
+        "PATH": os.environ.get("PATH", "/usr/bin:/bin"),
+        "RESTIC_REPOSITORY": "s3:"
+        + data["S3_ENDPOINT"].rstrip("/")
+        + "/"
+        + data["S3_BUCKET"]
+        + "/cliproxy-fleet/restic",
+        "RESTIC_PASSWORD": data["CLIPROXY_FLEET_RESTIC_PASSWORD"],
+        "AWS_ACCESS_KEY_ID": data["S3_ACCESS_KEY_ID"],
+        "AWS_SECRET_ACCESS_KEY": data["S3_SECRET_ACCESS_KEY"],
+        "AWS_DEFAULT_REGION": data["S3_REGION"],
+    }
+
+
+def client(data):
+    environment = restic_environment(data)
+
+    def run(*args):
+        if not args or args[0] not in ("snapshots", "dump", "restore"):
+            raise ValueError("Only read-only cloud operations are permitted")
+        return subprocess.run(
+            ["restic", "-o", "s3.bucket-lookup=path", "--no-cache", "--no-lock", *args],
+            env=environment,
+            check=True,
+            capture_output=True,
+            text=True,
+            timeout=180,
+        ).stdout
+
+    return run
+
+
+def latest(restic):
+    snapshots = json.loads(
+        restic(
+            "snapshots",
+            "--json",
+            "--host",
+            "cliproxy-primary",
+            "--tag",
+            "cliproxy-checkpoint-v1",
+        )
+    )
+    if not snapshots:
+        raise ValueError("No cloud checkpoint")
+    snapshot = max(snapshots, key=lambda value: timestamp(value["time"]))
+    if not re.fullmatch(r"[0-9a-f]{64}", snapshot["id"]):
+        raise ValueError("Invalid checkpoint identifier")
+    paths = snapshot["paths"]
+    if (
+        len(paths) != 1
+        or not pathlib.PurePosixPath(paths[0]).is_absolute()
+        or ".." in pathlib.PurePosixPath(paths[0]).parts
+    ):
+        raise ValueError("Invalid checkpoint path")
+    return snapshot
+
+
+def timestamp(value):
+    parsed = datetime.datetime.fromisoformat(value.replace("Z", "+00:00"))
+    if parsed.tzinfo is None:
+        raise ValueError("Timestamp must include timezone")
+    return parsed
+
+
+def fresh(value, seconds, now=None):
+    now = now or datetime.datetime.now(datetime.timezone.utc)
+    age = (now - timestamp(value)).total_seconds()
+    if not -300 <= age <= seconds:
+        raise ValueError("Cloud status is stale or future-dated")
+
+
+def client_key(token):
+    """Read only the client API key; independent of the backup fields in settings()."""
+    status, body = request(shared_url(), {"X-Vault-Token": token})
+    if status != 200:
+        raise ValueError("Vault read failed")
+    value = json.loads(body)["data"]["data"].get("CLIPROXY_FLEET_CLIENT_API_KEY")
+    if not isinstance(value, str) or not value.strip():
+        raise ValueError("Vault has no client API key")
+    return value.strip()