@crvouga/dev-fleet 0.0.0-stage → 0.2.0

package/cliproxy/scripts/recover_node.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+"""Prepare a complete offline node cache atomically; never stop/start the live stack."""
+
+import sys
+
+sys.dont_write_bytecode = True
+
+import pathlib as _pathlib
+
+sys.path.insert(0, str(_pathlib.Path(__file__).resolve().parent))
+import cliproxy_host
+
+import argparse
+import ctypes
+import datetime
+import json
+import os
+import pathlib
+import stat
+import shutil
+import subprocess
+import tempfile
+import checkpoint
+import cloud_state
+
+
+def publish(candidate, destination):
+    """Atomic directory publication with no replacement, including races."""
+    library = ctypes.CDLL(None, use_errno=True)
+    if cliproxy_host.system() == "macos":
+        # renamex_np(from, to, RENAME_EXCL)
+        rename = library.renamex_np
+        rename.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_uint]
+        arguments = [os.fsencode(candidate), os.fsencode(destination), 0x4]
+    else:
+        # renameat2(AT_FDCWD, from, AT_FDCWD, to, RENAME_NOREPLACE)
+        rename = library.renameat2
+        rename.argtypes = [
+            ctypes.c_int,
+            ctypes.c_char_p,
+            ctypes.c_int,
+            ctypes.c_char_p,
+            ctypes.c_uint,
+        ]
+        arguments = [-100, os.fsencode(candidate), -100, os.fsencode(destination), 1]
+    rename.restype = ctypes.c_int
+    if rename(*arguments) != 0:
+        raise OSError(ctypes.get_errno(), "Recovery destination could not be published")
+
+
+def verify_source(payload):
+    subprocess.run(
+        [
+            sys.executable,
+            "-B",
+            str(pathlib.Path(__file__).parent / "restore-verify"),
+            str(payload),
+        ],
+        check=True,
+        capture_output=True,
+        timeout=60,
+    )
+
+
+def recover(
+    destination,
+    token_path,
+    restic_factory=cloud_state.client,
+    read_settings=cloud_state.settings,
+):
+    # Refuse even an empty existing directory: no user data or deployment can be overwritten.
+    live_root = cliproxy_host.default_root().resolve()
+    canonical_destination = destination.parent.resolve() / destination.name
+    if live_root.exists() and canonical_destination.is_relative_to(live_root):
+        raise ValueError("Recovery destination must be outside the existing deployment")
+    if destination.exists() or destination.is_symlink():
+        raise ValueError("Recovery destination already exists")
+    if not destination.parent.is_dir():
+        raise ValueError("Recovery parent directory must exist")
+    metadata = token_path.lstat()
+    if not stat.S_ISREG(metadata.st_mode) or metadata.st_mode & 0o077:
+        raise ValueError("Vault token must be a private regular file")
+    token = token_path.read_text().strip()
+    if not token:
+        raise ValueError("Existing Vault login required")
+    data = read_settings(token)
+    restic = restic_factory(data)
+    snapshot = cloud_state.latest(restic)
+    os.umask(0o077)
+    with tempfile.TemporaryDirectory(
+        prefix=".cliproxy-recover-", dir=destination.parent
+    ) as directory:
+        staging = pathlib.Path(directory)
+        restored = staging / "restored"
+        restic("restore", snapshot["id"], "--target", str(restored), "--verify")
+        payload = checkpoint.locate(restored)
+        # Validate the source manifest against trusted Git objects, never run restored code.
+        verify_source(payload)
+        repository = json.loads((payload / "state/repository.json").read_text())
+        if repository != {
+            key: data[key] for key in ("S3_ENDPOINT", "S3_BUCKET", "S3_REGION")
+        }:
+            raise ValueError("Recovered repository identity differs from Vault")
+        import yaml
+
+        config = yaml.safe_load((payload / "state/runtime/config.yaml").read_text())
+        if data["CLIPROXY_FLEET_CLIENT_API_KEY"] not in config["access"]["api-keys"]:
+            raise ValueError("Checkpoint client key differs from current Vault key")
+        candidate = staging / "candidate"
+        state = candidate / "state"
+        state.mkdir(parents=True, mode=0o700)
+        checkpoint.hydrate(payload, state)
+        secrets = state / "secrets"
+        secrets.mkdir(mode=0o700)
+        values = {
+            "vault-token": token,
+            "client-key": data["CLIPROXY_FLEET_CLIENT_API_KEY"],
+            "management-key": data["CLIPROXY_FLEET_MANAGEMENT_KEY"],
+            "restic-password": data["CLIPROXY_FLEET_RESTIC_PASSWORD"],
+            "recovery-confirmed": "Recovered independently from Vault.",
+        }
+        for name, value in values.items():
+            file = secrets / name
+            file.write_text(value + "\n")
+            file.chmod(0o600)
+        (state / "status").mkdir(mode=0o700)
+        gc_status = payload / "state/status/gc-success.json"
+        if gc_status.is_file():
+            shutil.copyfile(gc_status, state / "status/gc-success.json")
+        source = json.loads((payload / "INSTALLATION.json").read_text())
+        (candidate / "RECOVERY.json").write_text(
+            json.dumps(
+                {
+                    "snapshot": snapshot["id"],
+                    "source_revision": source["revision"],
+                    "recovered_at": datetime.datetime.now(
+                        datetime.timezone.utc
+                    ).isoformat(),
+                    "activated": False,
+                }
+            )
+            + "\n"
+        )
+        publish(candidate, destination)
+    return snapshot["id"]
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "destination",
+        type=pathlib.Path,
+        help="New, nonexistent deployment/cache directory",
+    )
+    parser.add_argument(
+        "--vault-token-file",
+        type=pathlib.Path,
+        default=cliproxy_host.default_root()
+        / "state/secrets/vault-token",
+    )
+    args = parser.parse_args()
+    # Do not resolve the final component: an existing dangling symlink must be refused.
+    destination = args.destination.absolute()
+    recover(destination, args.vault_token_file)
+    print("Verified cloud recovery prepared at", destination)
+    print(
+        "No services or provider identities activated. RECOVERY.json records the source revision."
+    )
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception:
+        raise SystemExit(
+            "Offline recovery failed; live deployment unchanged. Credentials and upstream responses suppressed."
+        ) from None

package/cliproxy/scripts/restore-verify

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+"""Validate restored data with trusted repository code, without activating credentials."""
+
+import hashlib
+import importlib.util
+import json
+import pathlib
+import subprocess
+import sys
+
+sys.dont_write_bytecode = True
+root = pathlib.Path(sys.argv[1]).resolve()
+source = pathlib.Path(__file__).resolve().parent
+if not (root / "state/runtime/config.yaml").is_file():
+    import checkpoint
+    root = checkpoint.locate(root)
+if (root / "CHECKPOINT.json").exists():
+    import checkpoint
+    checkpoint.validate(root)
+if not (root / "state/runtime/config.yaml").is_file():
+    sys.exit("Pass the restored deployment root containing state/runtime/config.yaml.")
+subprocess.run(
+    [
+        sys.executable,
+        "-B",
+        str(source / "validate-config"),
+        str(root / "state/runtime/config.yaml"),
+    ],
+    check=True,
+)
+count = 0
+for file in (root / "state/runtime/auths").glob("*.json"):
+    try:
+        json.loads(file.read_text())
+        count += 1
+    except Exception:
+        sys.exit("Restored credential JSON is invalid; contents suppressed.")
+manifest = root / "INSTALLATION.json"
+if manifest.exists():
+    metadata = json.loads(manifest.read_text())
+    spec = importlib.util.spec_from_file_location("install", source / "install.py")
+    installer = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(installer)
+    _, files = installer.source_files(source.parents[1], metadata["revision"])
+    expected = {
+        name: hashlib.sha256(data).hexdigest() for name, (_, data) in files.items()
+    }
+    if expected != metadata["files"]:
+        sys.exit("Restored source manifest differs from its Git commit.")
+    locks = json.loads(files["image.lock.json"][1])
+else:
+    locks = json.loads((root / "image.lock.json").read_text())
+assert all("@sha256:" in image for image in locks.values())
+assert (root / "state/runtime/static/management.html").is_file()
+print(
+    "Restored config, Git source metadata, dashboard and",
+    count,
+    "credential files validated offline.",
+)
+print("No restored credentials or code were activated.")

package/cliproxy/scripts/retention.py

@@ -0,0 +1,144 @@
+"""Bound disposable history. Call only while holding state/status/operation.lock."""
+
+import datetime
+import json
+import pathlib
+import re
+import shutil
+import stat
+import subprocess
+import sys
+import time
+
+sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
+import cliproxy_host
+
+
+def forget(restic):
+    # Do not group by random staging paths or format-version tags: both grow forever.
+    restic(
+        "forget",
+        "--tag",
+        "cliproxy-fleet",
+        "--host",
+        "cliproxy-primary",
+        "--group-by",
+        "host",
+        "--keep-last",
+        "96",
+        "--keep-daily",
+        "7",
+        "--keep-weekly",
+        "5",
+        "--keep-monthly",
+        "12",
+    )
+
+
+def prune(restic, now=None):
+    now = now or datetime.datetime.now(datetime.timezone.utc)
+    result = restic(
+        "snapshots",
+        "--json",
+        "--tag",
+        "cliproxy-checkpoint-v1",
+        "--host",
+        "cliproxy-primary",
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+    snapshots = json.loads(result.stdout)
+    times = [
+        datetime.datetime.fromisoformat(item["time"].replace("Z", "+00:00"))
+        for item in snapshots
+    ]
+    if not times or not 0 <= (now - max(times)).total_seconds() <= 3600:
+        raise ValueError(
+            "A successful cloud checkpoint within one hour is required for GC"
+        )
+    restic(
+        "check"
+    )  # Never delete history from a repository already failing integrity checks.
+    forget(restic)
+    # Physical collection applies only inside this dedicated Restic repository.
+    # No repack cap: a cap could let garbage arrive faster than it is reclaimed.
+    restic("prune", "--max-unused", "5%")
+    restic("check")
+
+
+def staging(root, now=None):
+    now = time.time() if now is None else now
+    recovery = root / "recovery"
+    if not recovery.exists():
+        return 0
+    if recovery.is_symlink():
+        raise ValueError("Recovery directory must not be a link")
+    removed = 0
+    for path in recovery.iterdir():
+        if not re.fullmatch(r"(?:checkpoint|restore)-[a-z0-9_]{8}", path.name):
+            continue
+        metadata = path.lstat()
+        if stat.S_ISDIR(metadata.st_mode) and now - metadata.st_mtime > 86400:
+            # Python's fd-based rmtree does not traverse symlinks inside the tree.
+            if not shutil.rmtree.avoids_symlink_attacks:
+                raise ValueError("Safe directory deletion unavailable")
+            shutil.rmtree(path)
+            removed += 1
+    return removed
+
+
+def revisions(root, references=None):
+    """Retain current and previous installed Git revisions; never force removal."""
+    current = json.loads((root / "INSTALLATION.json").read_text())
+    previous = root / "INSTALLATION.previous.json"
+    keep = {current["revision"]}
+    if previous.exists():
+        keep.add(json.loads(previous.read_text())["revision"])
+    parent = root / "revisions"
+    if references is None:
+        home = cliproxy_host.runtime_home()
+        units = cliproxy_host.backend().unit_directory
+        references = [
+            *(home / ".local/bin").glob("*-fleet"),
+            *units.glob("cliproxy-*"),
+            *units.glob("dev.devfleet.cliproxy-*"),
+        ]
+    for reference in references:
+        if reference.is_symlink():
+            try:
+                keep.add(reference.resolve().relative_to(parent.resolve()).parts[0])
+            except ValueError:
+                pass
+    active = parent / current["revision"]
+    if parent.is_symlink() or active.is_symlink() or not active.is_dir():
+        raise ValueError("Invalid installed worktree directory")
+
+    def git(path, *args):
+        return subprocess.check_output(
+            [
+                "git",
+                "--no-optional-locks",
+                "-c",
+                "safe.directory=" + str(path),
+                "-C",
+                str(path),
+                *args,
+            ],
+            stderr=subprocess.PIPE,
+            text=True,
+        ).strip()
+
+    removed = 0
+    for path in parent.iterdir():
+        if path.name in keep or not re.fullmatch(r"[0-9a-f]{40}", path.name):
+            continue
+        if path.is_symlink() or not path.is_dir():
+            continue
+        if git(path, "rev-parse", "HEAD") != path.name:
+            raise ValueError("Unexpected worktree revision")
+        if git(path, "status", "--porcelain", "--untracked-files=all", "--ignored"):
+            raise ValueError("Retired worktree has changes; refusing deletion")
+        git(active, "worktree", "remove", str(path))
+        removed += 1
+    return removed

package/cliproxy/scripts/service-start

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+# Invoked as INSTALL_ROOT/scripts/service-start by systemd (Linux) or launchd (macOS).
+cd "$(dirname "$0")/.."
+export CLIPROXY_ROOT="$PWD"
+./scripts/audit.py
+exec ./scripts/ops start

package/cliproxy/scripts/validate-config

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+import sys
+
+sys.dont_write_bytecode = True
+
+import pathlib as _pathlib
+
+sys.path.insert(0, str(_pathlib.Path(__file__).resolve().parent))
+import cliproxy_host
+
+import pathlib, sys, yaml, os
+
+arguments = [a for a in sys.argv[1:] if not a.startswith("--")]
+require_fill_first = "--require-fill-first" in sys.argv[1:]
+p = (
+    pathlib.Path(arguments[0])
+    if arguments
+    else cliproxy_host.default_root()
+    / "state/runtime/config.yaml"
+)
+try:
+    c = yaml.safe_load(p.read_text())
+
+    def require(condition):
+        if not condition:
+            raise ValueError("Unsafe configuration")
+
+    require(c["config-version"] == 8)
+    require(c["server"]["port"] == 8317)
+    require(c["oauth"]["auth-dir"] == "/state/auths")
+    management = c["management"]["secret-key"]
+    require(
+        isinstance(management, str)
+        and len(management) >= 32
+        and "REPLACE" not in management
+    )
+    keys = c["access"]["api-keys"]
+    require(
+        isinstance(keys, list)
+        and bool(keys)
+        and all(
+            isinstance(k, str) and len(k) >= 32 and "REPLACE" not in k for k in keys
+        )
+    )
+    require(management not in keys)
+    require(c["routing"]["strategy"] in ("round-robin", "weighted-round-robin", "fill-first"))
+    # Candidate configurations must keep the fill-first policy; the running config is only
+    # reported on drift (see scripts/monitor), so an older deployment still starts.
+    require(not require_fill_first or c["routing"]["strategy"] == "fill-first")
+    require(c.get("quota-exceeded", {}).get("switch-preview-model") is False)
+    require(c["observability"]["logs"]["debug"] is False)
+    require(c["observability"]["logs"]["request-log"] is False)
+    require(c["plugins"]["enabled"] is False)
+except Exception:
+    sys.exit("Configuration failed YAML/security validation; values suppressed.")
+print("YAML and deployment safety invariants passed (runtime staging still required).")

package/cliproxy/scripts/vault-access.py

@@ -0,0 +1,57 @@
+"""Vault API helpers. Never print response bodies, tokens, or credentials."""
+
+import sys
+
+sys.dont_write_bytecode = True
+
+import pathlib as _pathlib
+
+sys.path.insert(0, str(_pathlib.Path(__file__).resolve().parent))
+import cliproxy_host
+import fleet_config
+
+
+import json, pathlib, urllib.request, os
+
+ROOT = cliproxy_host.default_root().resolve()
+
+
+def request(path, token=None, data=None):
+    headers = {"Content-Type": "application/json"}
+    if token:
+        headers["X-Vault-Token"] = token
+    req = urllib.request.Request(
+        fleet_config.vault()["address"] + "/v1/" + path,
+        headers=headers,
+        data=json.dumps(data).encode() if data is not None else None,
+    )
+    with urllib.request.urlopen(req, timeout=30) as r:
+        return json.load(r)
+
+
+def token():
+    return (ROOT / "state/secrets/vault-token").read_text().strip()
+
+
+def credentials():
+    data = request(fleet_config.vault()["shared"], token())["data"]["data"]
+    fields = [
+        "S3_ENDPOINT",
+        "S3_REGION",
+        "S3_BUCKET",
+        "S3_ACCESS_KEY_ID",
+        "S3_SECRET_ACCESS_KEY",
+    ]
+    if any(not data.get(k) for k in fields):
+        raise ValueError("Missing S3 field")
+    return {k: data[k] for k in fields}
+
+
+def renew():
+    t = token()
+    meta = request("auth/token/lookup-self", t)["data"]
+    if meta.get("ttl", 0) < 3600:
+        if not meta.get("renewable"):
+            raise ValueError("Token expiry requires login")
+        request("auth/token/renew-self", t, {})
+    return meta.get("ttl", 0)

package/cliproxy/scripts/vault-backup-login

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+import sys
+
+sys.dont_write_bytecode = True
+
+import pathlib as _pathlib
+
+sys.path.insert(0, str(_pathlib.Path(__file__).resolve().parent))
+import cliproxy_host
+import fleet_config
+
+import getpass, importlib.util, os, pathlib, sys
+
+root = cliproxy_host.default_root().resolve()
+spec = importlib.util.spec_from_file_location(
+    "vault_access", root / "scripts/vault-access.py"
+)
+v = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(v)
+os.umask(0o077)
+for directory in [root / "state", root / "state/secrets", root / "state/status"]:
+    directory.mkdir(mode=0o700, parents=True, exist_ok=True)
+    if os.geteuid() == 0:
+        os.chown(directory, 1000, 1000)
+try:
+    password = getpass.getpass("Vault password for " + fleet_config.vault()["user"] + " (hidden): ")
+    token = v.request("auth/userpass/login/" + fleet_config.vault()["user"], data={"password": password})[
+        "auth"
+    ]["client_token"]
+    password = None
+    data = v.request(fleet_config.vault()["shared"], token)["data"]["data"]
+    fields = [
+        "S3_ENDPOINT",
+        "S3_REGION",
+        "S3_BUCKET",
+        "S3_ACCESS_KEY_ID",
+        "S3_SECRET_ACCESS_KEY",
+    ]
+    if any(not data.get(k) for k in fields):
+        raise ValueError("Missing required S3 contract fields")
+    f = root / "state/secrets/vault-token"
+    new = f.with_suffix(".new")
+    new.write_text(token + "\n")
+    new.chmod(0o600)
+    os.replace(new, f)
+    if os.geteuid() == 0:
+        os.chown(f, 1000, 1000)
+    (root / "state/secrets/s3.json").unlink(missing_ok=True)
+    print(
+        "Saved Vault token securely. Each backup will fetch live S3 credentials; cached S3 credentials removed."
+    )
+except Exception:
+    sys.exit(
+        "Vault login/read failed. Check credentials and access to the shared document; response suppressed."
+    )

package/cliproxy/scripts/vault-recovery

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+"""Read existing recovery secrets; all Vault writes belong to the OpenTofu root."""
+
+import sys
+
+sys.dont_write_bytecode = True
+
+import pathlib as _pathlib
+
+sys.path.insert(0, str(_pathlib.Path(__file__).resolve().parent))
+import cliproxy_host
+import fleet_config
+
+
+import hmac
+import importlib.util
+import os
+import pathlib
+import sys
+
+root = cliproxy_host.default_root().resolve()
+spec = importlib.util.spec_from_file_location(
+    "vault_access", root / "scripts/vault-access.py"
+)
+vault = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(vault)
+os.umask(0o077)
+try:
+    if sys.argv[1:] not in [["recover"], ["verify"]]:
+        sys.exit(
+            "Usage: vault-recovery recover|verify. Manage secret values through bun run vault:edit."
+        )
+    secret = vault.request(fleet_config.vault()["shared"], vault.token())["data"]["data"][
+        "CLIPROXY_FLEET_RESTIC_PASSWORD"
+    ]
+    local = root / "state/secrets/restic-password"
+    if sys.argv[1] == "recover":
+        if local.exists() and not hmac.compare_digest(
+            local.read_text().strip(), secret
+        ):
+            sys.exit("Local recovery password differs; refusing to overwrite it.")
+        local.write_text(secret + "\n")
+        local.chmod(0o600)
+        if os.geteuid() == 0:
+            os.chown(local, 1000, 1000)
+    if not hmac.compare_digest(local.read_text().strip(), secret):
+        sys.exit("Recovery password mismatch; values suppressed.")
+    marker = root / "state/secrets/recovery-confirmed"
+    marker.write_text("Verified existing independent Vault recovery copy.\n")
+    if os.geteuid() == 0:
+        os.chown(marker, 1000, 1000)
+    print("Independent Vault recovery copy verified; no secrets printed.")
+except Exception:
+    sys.exit(
+        "Vault recovery failed. Check login, registered key and read access; response suppressed."
+    )

package/cliproxy/scripts/vault-renew

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+import sys
+
+sys.dont_write_bytecode = True
+
+import pathlib as _pathlib
+
+sys.path.insert(0, str(_pathlib.Path(__file__).resolve().parent))
+import cliproxy_host
+
+import importlib.util, pathlib, sys, os
+
+root = cliproxy_host.default_root().resolve()
+spec = importlib.util.spec_from_file_location(
+    "vault_access", root / "scripts/vault-access.py"
+)
+v = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(v)
+try:
+    v.renew()
+    (root / "state/status/vault-failed").unlink(missing_ok=True)
+    print("Vault token valid; renewal checked.")
+except Exception:
+    f = root / "state/status/vault-failed"
+    f.write_text("Vault login/renewal required.\n")
+    if os.geteuid() == 0:
+        os.chown(f, 1000, 1000)
+    sys.exit(
+        "Vault token unavailable, expired, or unable to renew. Run vault-backup-login."
+    )

package/cliproxy/systemd/cliproxy-backup-check.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Verify CLIProxyAPI encrypted backup integrity
+After=network-online.target
+[Service]
+Environment=PYTHONDONTWRITEBYTECODE=1
+Type=oneshot
+ExecStart=/opt/cliproxy-fleet/scripts/backup check
+UMask=0077
+Nice=10
+TimeoutStartSec=4h

package/cliproxy/systemd/cliproxy-backup-check.timer

@@ -0,0 +1,8 @@
+[Unit]
+Description=Weekly Restic repository and 10 percent data integrity check
+[Timer]
+OnCalendar=Sun *-*-* 05:00:00
+RandomizedDelaySec=30m
+Persistent=true
+[Install]
+WantedBy=timers.target

package/cliproxy/systemd/cliproxy-backup.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Consistent encrypted CLIProxyAPI backup to independent R2 storage
+Requires=docker.service
+After=docker.service network-online.target
+[Service]
+Environment=PYTHONDONTWRITEBYTECODE=1
+Type=oneshot
+WorkingDirectory=/opt/cliproxy-fleet
+ExecStart=/opt/cliproxy-fleet/scripts/backup backup
+UMask=0077
+Nice=10
+IOSchedulingClass=idle
+TimeoutStartSec=1h