@astrofoundry/pi-astro 0.33.2 → 0.34.0

package/specialists/entry/remote/spc-arcane-op.py

@@ -0,0 +1,685 @@
+#!/usr/bin/python3
+"""Root-side operations for the arcane specialist's forced-command key on VM 100.
+
+Installed root-owned as /usr/local/sbin/spc-arcane-op. The forced command
+spc-arcane-host-entry runs it through one exact sudoers line for user
+spc-arcane, so this file is the only privileged surface of that key. Every
+operation has a fixed name and strictly checked arguments; the read-only
+checks run the reviewed arcane-ops probes from /usr/local/lib/spc-arcane and
+print only the fields those probes allow. Commands, image digests, and
+database names come from this file, never from the caller.
+"""
+
+import gzip
+import hashlib
+import importlib.util
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tarfile
+import time
+import urllib.error
+import urllib.request
+from datetime import datetime, timezone
+from pathlib import Path
+
+PROBES = Path("/usr/local/lib/spc-arcane")
+DOCKER = "/usr/bin/docker"
+COMPOSE_DIR = Path("/opt/docker/arcane")
+COMPOSE_FILE = COMPOSE_DIR / "compose.yaml"
+MANAGER = "arcane"
+MANAGER_IMAGE = "ghcr.io/getarcaneapp/manager"
+MANAGER_DATA = Path("/var/lib/docker/volumes/arcane_arcane-data/_data")
+UPGRADES = Path("/var/backups/arcane-upgrades")
+DUMPS = Path("/var/backups/spc-arcane")
+KEEP_DUMPS = 3
+LOCAL_API = "http://127.0.0.1:3552"
+
+# PostgreSQL stacks with plain pg_dump/psql dumps: database container, role and
+# database (both named like the stack's POSTGRES_USER and POSTGRES_DB), the one
+# app container stopped during a restore, and a table whose row count shows the
+# data arrived (None when no table is known).
+DATABASES = {
+    "mealie": {"db": "mealie-db", "user": "mealie", "database": "mealie", "app": "mealie", "table": "recipes"},
+    "paperless": {"db": "paperless-db", "user": "paperless", "database": "paperless", "app": "paperless", "table": "documents_document"},
+}
+
+# Tracked host files the audit compares with the repository's host/ folder.
+HOST_FILES = (
+    ("arcane-backup.sh", "/usr/local/bin/arcane-backup.sh"),
+    ("arcane-backup.service", "/etc/systemd/system/arcane-backup.service"),
+    ("arcane-backup.timer", "/etc/systemd/system/arcane-backup.timer"),
+    ("restic-check.sh", "/usr/local/bin/restic-check.sh"),
+    ("restic-check.service", "/etc/systemd/system/restic-check.service"),
+    ("restic-check.timer", "/etc/systemd/system/restic-check.timer"),
+    ("media-watchdog.sh", "/usr/local/bin/media-watchdog.sh"),
+    ("media-watchdog.service", "/etc/systemd/system/media-watchdog.service"),
+    ("media-watchdog.timer", "/etc/systemd/system/media-watchdog.timer"),
+    ("nas-media-check.py", "/usr/local/sbin/nas-media-check"),
+    ("nas-media-check.service", "/etc/systemd/system/nas-media-check.service"),
+    ("nas-media-check.timer", "/etc/systemd/system/nas-media-check.timer"),
+    ("nas-media-check.logrotate", "/etc/logrotate.d/nas-media-check"),
+    ("grimoire-api-firewall.sh", "/usr/local/sbin/grimoire-api-firewall.sh"),
+    ("grimoire-api-firewall.service", "/etc/systemd/system/grimoire-api-firewall.service"),
+    ("spc-backup-arcane-entry", "/usr/local/sbin/spc-backup-arcane-entry"),
+    ("spc-arcane-host-entry", "/usr/local/sbin/spc-arcane-host-entry"),
+    ("spc-arcane-op", "/usr/local/sbin/spc-arcane-op"),
+    ("spc-arcane-sudoers", "/etc/sudoers.d/spc-arcane"),
+)
+UNITS = tuple(name for name, _ in HOST_FILES if name.endswith((".service", ".timer")))
+LOOPBACK_CHECKS = {
+    "grimoire": "http://127.0.0.1:8001/healthz",
+    "mealie": "http://127.0.0.1:8083/api/recipes?page=1&perPage=1",
+}
+MEALIE_API = "http://127.0.0.1:8083/api/recipes?page=1&perPage=1"
+OBJECT_ROOTS = {"grimoire": "/var/lib/grimoire/object-store"}
+CAPACITY_PATHS = {"containerd": "/var/lib/containerd", "grimoire": "/var/lib/grimoire/object-store"}
+STORAGE_AREAS = (
+    "root", "var", "var-lib", "logs", "docker", "volumes", "container-logs", "containerd", "grimoire",
+    "supabase", "nzb-scratch", "nzb-complete", "opt", "cache", "backups", "backup-staging", "backup-grimoire",
+)
+
+NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")
+UTC = re.compile(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$")
+STAMP = re.compile(r"^[0-9]{8}T[0-9]{6}Z$")
+DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$")
+VERSION = re.compile(r"^v?[0-9]{1,4}\.[0-9]{1,4}\.[0-9]{1,4}$")
+IMDB = re.compile(r"^tt[0-9]{1,12}$")
+UUID = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")
+INTEGER = re.compile(r"^[0-9]{1,9}$")
+OBJECT_PATH = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_./-]{0,255}$")
+
+
+class Refused(Exception):
+    """A request this file does not accept; the caller sees usage text and exit 2."""
+
+
+class Failed(Exception):
+    """An operation that started and did not complete; exit 1."""
+
+
+def emit(value):
+    print(json.dumps(value, sort_keys=True))
+
+
+def need(pattern, value, what):
+    if not isinstance(value, str) or pattern.fullmatch(value) is None:
+        raise Refused(what)
+    return value
+
+
+def arity(args, low, high, usage):
+    if not low <= len(args) <= high:
+        raise Refused(usage)
+
+
+def utc_stamp():
+    return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+
+
+def run(argv, timeout, cwd=None, stdin=None, stdout=None):
+    """Runs a fixed argument list; output that could hold data is never returned to the caller."""
+    try:
+        return subprocess.run(argv, cwd=cwd, stdin=stdin, stdout=stdout if stdout is not None else subprocess.PIPE,
+                              stderr=subprocess.PIPE, timeout=timeout, check=False)
+    except subprocess.TimeoutExpired:
+        raise Failed(f"{Path(argv[0]).name} {argv[1] if len(argv) > 1 else ''} timed out after {timeout} s") from None
+
+
+def must(argv, timeout, what, cwd=None):
+    result = run(argv, timeout, cwd=cwd)
+    if result.returncode != 0:
+        raise Failed(f"{what} failed (exit {result.returncode}); its output was withheld")
+    return result.stdout.decode("utf-8", "replace")
+
+
+def sha256_file(path):
+    digest = hashlib.sha256()
+    with open(path, "rb") as source:
+        for block in iter(lambda: source.read(1 << 20), b""):
+            digest.update(block)
+    return digest.hexdigest()
+
+
+def root_dir(path):
+    """Creates a directory only root can read; refuses one that already exists as a link or non-directory."""
+    if path.is_symlink() or (path.exists() and not path.is_dir()):
+        raise Failed(f"{path} is not a plain directory")
+    path.mkdir(mode=0o700, parents=True, exist_ok=True)
+    os.chown(path, 0, 0)
+    os.chmod(path, 0o700)
+    return path
+
+
+# ---------------------------------------------------------------- read-only probes
+
+
+def probe(module, request):
+    """Runs one reviewed arcane-ops probe; it prints its own allowlisted JSON."""
+    path = PROBES / f"{module}.py"
+    spec = importlib.util.spec_from_file_location(f"spc_arcane_{module}", path)
+    if spec is None or spec.loader is None:
+        raise Failed(f"probe {module} is not installed")
+    loaded = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(loaded)
+    loaded.run(request)
+
+
+def window_args(args, usage):
+    since, until = args
+    need(UTC, since, usage)
+    need(UTC, until, usage)
+    return since, until
+
+
+def op_container_memory(args):
+    arity(args, 1, 1, "container-memory <name>")
+    probe("resource_probe", {"kind": "container_memory", "name": need(NAME, args[0], "container-memory <name>")})
+
+
+def op_container_stats(args):
+    arity(args, 0, 8, "container-stats [name...] (at most 8)")
+    probe("resource_probe", {"kind": "container_stats", "names": [need(NAME, a, "container-stats [name...]") for a in args]})
+
+
+def op_container_logs(args):
+    usage = "container-logs <name> <since> <until> [docker|bazarr-file] (UTC like 2026-09-06T12:15:00Z)"
+    arity(args, 3, 4, usage)
+    name = need(NAME, args[0], usage)
+    since, until = window_args(args[1:3], usage)
+    source = args[3] if len(args) == 4 else "docker"
+    if source not in ("docker", "bazarr-file") or (source == "bazarr-file" and name != "bazarr"):
+        raise Refused("the bazarr-file source is only available for bazarr")
+    probe("incident_probe", {"kind": "container_logs", "name": name, "since": since, "until": until, "source": source})
+
+
+def op_container_oom(args):
+    usage = "container-oom <name> <since> <until>"
+    arity(args, 3, 3, usage)
+    name = need(NAME, args[0], usage)
+    since, until = window_args(args[1:3], usage)
+    probe("incident_probe", {"kind": "container_oom", "name": name, "since": since, "until": until})
+
+
+def op_kernel_oom(args):
+    usage = "kernel-oom <since> <until>"
+    arity(args, 2, 2, usage)
+    since, until = window_args(args, usage)
+    probe("kernel_probe", {"kind": "kernel_oom", "since": since, "until": until})
+
+
+def op_host_audit(args):
+    arity(args, 0, 0, "host-audit")
+    probe("host_probe", {"kind": "audit", "files": [{"name": n, "path": p} for n, p in HOST_FILES], "units": list(UNITS)})
+
+
+def op_host_capacity(args):
+    arity(args, 0, 0, "host-capacity")
+    probe("resource_probe", {"kind": "host_capacity", "paths": CAPACITY_PATHS})
+
+
+def op_host_storage(args):
+    arity(args, 0, 1, "host-storage [area]")
+    area = args[0] if args else "root"
+    if area not in STORAGE_AREAS:
+        raise Refused("host-storage area: " + ", ".join(STORAGE_AREAS))
+    probe("storage_probe", {"kind": "host_storage", "area": area})
+
+
+def op_docker_usage(args):
+    arity(args, 0, 0, "docker-usage")
+    probe("storage_probe", {"kind": "docker_usage"})
+
+
+def op_service_check(args):
+    arity(args, 1, 1, "service-check grimoire|mealie")
+    if args[0] not in LOOPBACK_CHECKS:
+        raise Refused("service-check grimoire|mealie (public routes are edge's route-probe)")
+    probe("host_probe", {"kind": "http", "url": LOOPBACK_CHECKS[args[0]], "timeout": 10})
+
+
+def fixed_probe(module, kind):
+    def op(args):
+        arity(args, 0, 0, kind.replace("_", "-"))
+        probe(module, {"kind": kind})
+    return op
+
+
+def op_mealie_api(args):
+    arity(args, 0, 0, "mealie-api (the key arrives on stdin)")
+    key = sys.stdin.read(4096).strip()
+    if not re.fullmatch(r"[A-Za-z0-9._-]{16,2048}", key):
+        raise Refused("mealie-api needs the Mealie API key on stdin")
+    probe("mealie_probe", {"kind": "mealie_api", "url": MEALIE_API, "key": key})
+
+
+def op_radarr_interface(args):
+    arity(args, 0, 0, "radarr-interface")
+    probe("radarr_probe", {"kind": "radarr_api", "action": "interface", "host_config": True})
+
+
+def op_radarr_movie(args):
+    usage = "radarr-movie <tt…> [command-id]"
+    arity(args, 1, 2, usage)
+    request = {"kind": "radarr_api", "action": "movie", "host_config": True, "imdb": need(IMDB, args[0], usage)}
+    if len(args) == 2:
+        request["command_id"] = int(need(INTEGER, args[1], usage))
+    probe("radarr_probe", request)
+
+
+def op_queue_job(args):
+    arity(args, 1, 1, "queue-job <uuid>")
+    probe("queue_probe", {"kind": "queue_job", "job_id": need(UUID, args[0], "queue-job <uuid>")})
+
+
+def op_object_stat(args):
+    usage = "object-stat grimoire <relative path>"
+    arity(args, 2, 2, usage)
+    if args[0] not in OBJECT_ROOTS:
+        raise Refused(usage)
+    relative = need(OBJECT_PATH, args[1], usage)
+    parts = relative.split("/")
+    if any(part in ("", ".", "..") or part.startswith(".") for part in parts):
+        raise Refused("object path: no hidden files, empty parts, or parent traversal")
+    probe("host_probe", {"kind": "object", "root": OBJECT_ROOTS[args[0]], "relative": relative})
+
+
+# ---------------------------------------------------------------- manager upgrade
+
+
+def manager_container():
+    raw = must([DOCKER, "inspect", MANAGER], 30, "docker inspect arcane")
+    info = json.loads(raw)[0]
+    image_id = info.get("Image")
+    image = json.loads(must([DOCKER, "image", "inspect", image_id], 30, "docker image inspect"))[0]
+    return {
+        "status": info.get("State", {}).get("Status"),
+        "started_at": info.get("State", {}).get("StartedAt"),
+        "configured_image": info.get("Config", {}).get("Image"),
+        "image_id": image_id,
+        "repo_digests": [d for d in image.get("RepoDigests") or [] if d.startswith(MANAGER_IMAGE + "@")],
+        "revision": (image.get("Config", {}).get("Labels") or {}).get("org.opencontainers.image.revision"),
+        "platform": f"{image.get('Os')}/{image.get('Architecture')}",
+    }
+
+
+class NoRedirect(urllib.request.HTTPRedirectHandler):
+    def redirect_request(self, *args, **kwargs):
+        return None
+
+
+def local_opener():
+    return urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect())
+
+
+def local_version():
+    """The running manager's version from its unauthenticated /api/version, read as the host probe reads it."""
+    try:
+        with local_opener().open(LOCAL_API + "/api/version", timeout=10) as response:
+            body = json.loads(response.read(65536))
+        value = body.get("currentVersion") if isinstance(body, dict) else None
+        return value if isinstance(value, str) and VERSION.fullmatch(value) else None
+    except (urllib.error.URLError, OSError, ValueError):
+        return None
+
+
+def local_ui_status():
+    opener = local_opener()
+    try:
+        with opener.open(LOCAL_API + "/", timeout=10) as response:
+            return response.status
+    except urllib.error.HTTPError as error:
+        return error.code
+    except (urllib.error.URLError, OSError):
+        return None
+
+
+def container_snapshot():
+    ids = must([DOCKER, "ps", "-aq", "--no-trunc"], 30, "docker ps").split()
+    if not ids:
+        return []
+    raw = json.loads(must([DOCKER, "inspect", *ids], 60, "docker inspect"))
+    return sorted(
+        ({"id": c.get("Id"), "name": (c.get("Name") or "").lstrip("/"), "image": c.get("Image"),
+          "started_at": c.get("State", {}).get("StartedAt"), "status": c.get("State", {}).get("Status")} for c in raw),
+        key=lambda c: c["name"],
+    )
+
+
+def upgrade_dir(stamp, create=False):
+    need(STAMP, stamp, "stamp: YYYYMMDDTHHMMSSZ")
+    path = UPGRADES / stamp
+    if create:
+        if path.exists():
+            raise Failed(f"{path} already exists")
+        root_dir(UPGRADES)
+        return root_dir(path)
+    if path.is_symlink() or not path.is_dir():
+        raise Failed(f"no upgrade record {stamp}")
+    return path
+
+
+def write_private(path, data):
+    descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
+    with os.fdopen(descriptor, "wb") as target:
+        target.write(data)
+
+
+def compose_digest():
+    match = re.search(re.escape(MANAGER_IMAGE) + r"@(sha256:[0-9a-f]{64})", COMPOSE_FILE.read_text(encoding="utf-8"))
+    return match.group(1) if match else None
+
+
+def op_manager_state(args):
+    arity(args, 0, 0, "manager-state")
+    emit({"container": manager_container(), "compose_sha256": sha256_file(COMPOSE_FILE), "compose_digest": compose_digest(),
+          "version": local_version()})
+
+
+def op_manager_prepare(args):
+    """Before the pin commit: records the deployed Compose file, the manager, and every other container."""
+    arity(args, 1, 1, "manager-prepare <stamp>")
+    path = upgrade_dir(args[0], create=True)
+    write_private(path / "compose.before.yaml", COMPOSE_FILE.read_bytes())
+    state = {"container": manager_container(), "compose_sha256": sha256_file(COMPOSE_FILE), "version": local_version()}
+    write_private(path / "state.before.json", json.dumps(state, sort_keys=True).encode())
+    write_private(path / "containers.before.json", json.dumps(container_snapshot(), sort_keys=True).encode())
+    emit({"stamp": args[0], "path": str(path), **state})
+
+
+def op_manager_pull(args):
+    """After the sync deployed the pin: pulls exactly that digest before any outage."""
+    arity(args, 1, 1, "manager-pull <sha256:digest>")
+    digest = need(DIGEST, args[0], "manager-pull <sha256:digest>")
+    if compose_digest() != digest:
+        raise Failed("the deployed Compose file does not pin that digest yet")
+    must([DOCKER, "compose", "pull", MANAGER], 900, "docker compose pull arcane", cwd=COMPOSE_DIR)
+    ref = f"{MANAGER_IMAGE}@{digest}"
+    image = json.loads(must([DOCKER, "image", "inspect", ref], 30, "docker image inspect"))[0]
+    emit({"image": ref, "platform": f"{image.get('Os')}/{image.get('Architecture')}",
+          "revision": (image.get("Config", {}).get("Labels") or {}).get("org.opencontainers.image.revision")})
+
+
+def exclude_env(info):
+    name = Path(info.name).name
+    return None if name == ".env" or name.startswith(".env.") else info
+
+
+def op_manager_backup(args):
+    """Stops only the manager, archives its data volume without .env files, and checks the archive."""
+    arity(args, 1, 1, "manager-backup <stamp>")
+    path = upgrade_dir(args[0])
+    archive = path / "arcane-data.tar.gz"
+    if archive.exists():
+        raise Failed(f"{archive} already exists")
+    must([DOCKER, "compose", "stop", MANAGER], 120, "docker compose stop arcane", cwd=COMPOSE_DIR)
+    try:
+        descriptor = os.open(archive, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
+        with os.fdopen(descriptor, "wb") as raw, tarfile.open(fileobj=raw, mode="w:gz") as tar:
+            tar.add(str(MANAGER_DATA), arcname=".", filter=exclude_env)
+        with tarfile.open(archive, "r:gz") as tar:
+            members = len(tar.getmembers())
+        checksum = sha256_file(archive)
+        write_private(path / "SHA256SUMS", f"{checksum}  arcane-data.tar.gz\n".encode())
+        manifest = {"stamp": args[0], "archive": archive.name, "sha256": checksum, "bytes": archive.stat().st_size,
+                    "members": members, "source": str(MANAGER_DATA), "excluded": [".env", ".env.*"]}
+        write_private(path / "manifest.json", json.dumps(manifest, sort_keys=True).encode())
+    except Exception as error:
+        run([DOCKER, "compose", "start", MANAGER], 120, cwd=COMPOSE_DIR)
+        raise Failed(f"backup failed ({type(error).__name__}); the previous manager was started again") from None
+    emit(manifest)
+
+
+def op_manager_start(args):
+    arity(args, 0, 0, "manager-start")
+    must([DOCKER, "compose", "up", "-d", "--no-deps", "--pull", "never", MANAGER], 300, "docker compose up arcane", cwd=COMPOSE_DIR)
+    emit({"started": MANAGER})
+
+
+def snapshot_changes(before, after):
+    """Other containers that restarted, changed image, or vanished while only the manager was replaced."""
+    now = {c["name"]: c for c in after}
+    changed = []
+    for old in before:
+        if old["name"] == MANAGER:
+            continue
+        new = now.get(old["name"])
+        if new is None:
+            changed.append({"name": old["name"], "change": "missing"})
+        elif any(new[k] != old[k] for k in ("id", "image", "started_at", "status")):
+            changed.append({"name": old["name"], "change": "restarted or replaced"})
+    return changed
+
+
+def op_manager_verify(args):
+    usage = "manager-verify <stamp> <sha256:digest> <version>"
+    arity(args, 3, 3, usage)
+    digest = need(DIGEST, args[1], usage)
+    version = need(VERSION, args[2], usage).lstrip("v")
+    path = upgrade_dir(args[0])
+    container = None
+    for _ in range(30):
+        container = manager_container()
+        if container["status"] == "running" and local_version() is not None:
+            break
+        time.sleep(2)
+    observed_version = local_version()
+    before = json.loads((path / "containers.before.json").read_text(encoding="utf-8"))
+    changes = snapshot_changes(before, container_snapshot())
+    ui = local_ui_status()
+    checks = [
+        {"check": "running", "ok": container["status"] == "running", "observed": container["status"]},
+        {"check": "digest", "ok": f"{MANAGER_IMAGE}@{digest}" in container["repo_digests"], "observed": container["repo_digests"]},
+        {"check": "version", "ok": (observed_version or "").lstrip("v") == version, "observed": observed_version},
+        {"check": "local ui", "ok": ui == 200, "observed": ui},
+        {"check": "other containers unchanged", "ok": not changes, "observed": changes},
+    ]
+    emit({"ok": all(c["ok"] for c in checks), "checks": checks})
+
+
+def op_manager_rollback(args):
+    """R2 of the upgrade rollback: the recorded Compose file and the previous image, nothing else.
+
+    The data rollback (R3) replaces the manager's data volume, so it stays an
+    operator step in arcane/AGENTS.md "Upgrade rollback"; this tool only keeps
+    the pre-upgrade archive and its checksum for it.
+    """
+    arity(args, 1, 1, "manager-rollback <stamp>")
+    path = upgrade_dir(args[0])
+    before = path / "compose.before.yaml"
+    if not before.is_file():
+        raise Failed("no recorded compose.before.yaml")
+    # Copying into the existing file keeps its owner and mode, which the GitOps sync manages.
+    shutil.copyfile(before, COMPOSE_FILE)
+    must([DOCKER, "compose", "pull", MANAGER], 900, "docker compose pull arcane", cwd=COMPOSE_DIR)
+    must([DOCKER, "compose", "up", "-d", "--no-deps", MANAGER], 300, "docker compose up arcane", cwd=COMPOSE_DIR)
+    emit({"rolled_back": True, "image": compose_digest(), "data_archive": str(path / "arcane-data.tar.gz"),
+          "data": "unchanged; restoring the pre-upgrade data is the manual R3 step"})
+
+
+# ---------------------------------------------------------------- database dumps
+
+
+def database(name):
+    if name not in DATABASES:
+        raise Refused("stack: " + ", ".join(DATABASES))
+    return DATABASES[name]
+
+
+def psql(db, sql, database_name, timeout=120):
+    result = run([DOCKER, "exec", db["db"], "psql", "-U", db["user"], "-d", database_name, "-X", "-q", "-At",
+                  "-v", "ON_ERROR_STOP=1", "-c", sql], timeout)
+    if result.returncode != 0:
+        raise Failed(f"psql on {db['db']} failed (exit {result.returncode}); its output was withheld")
+    return result.stdout.decode("utf-8", "replace").strip()
+
+
+def counts(db):
+    tables = int(psql(db, "SELECT count(*) FROM information_schema.tables WHERE table_schema = 'public'", db["database"]))
+    rows = None
+    if db["table"] is not None and psql(db, f"SELECT to_regclass('public.{db['table']}') IS NOT NULL", db["database"]) == "t":
+        rows = int(psql(db, f"SELECT count(*) FROM public.{db['table']}", db["database"]))
+    return {"public_tables": tables, "table": db["table"], "rows": rows}
+
+
+def dumps_of(stack):
+    base = DUMPS / stack
+    if not base.is_dir():
+        return []
+    return sorted(p for p in base.iterdir() if p.is_dir() and not p.is_symlink() and STAMP.fullmatch(p.name))
+
+
+def dump(stack):
+    db = database(stack)
+    stamp = utc_stamp()
+    root_dir(DUMPS)
+    path = root_dir(root_dir(DUMPS / stack) / stamp)
+    target = path / f"{stack}.sql.gz"
+    before = counts(db)
+    descriptor = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
+    with os.fdopen(descriptor, "wb") as raw, gzip.GzipFile(fileobj=raw, mode="wb") as out:
+        process = subprocess.Popen([DOCKER, "exec", db["db"], "pg_dump", "-U", db["user"], "-d", db["database"]],
+                                   stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+        shutil.copyfileobj(process.stdout, out)
+        code = process.wait(timeout=3600)
+    if code != 0:
+        raise Failed(f"pg_dump of {stack} failed (exit {code}); {target} is incomplete and has no manifest")
+    with gzip.open(target, "rb") as check:
+        while check.read(1 << 20):
+            pass
+    checksum = sha256_file(target)
+    write_private(path / "SHA256SUMS", f"{checksum}  {target.name}\n".encode())
+    info = {"stack": stack, "stamp": stamp, "file": str(target), "bytes": target.stat().st_size, "sha256": checksum, "counts": before}
+    write_private(path / "manifest.json", json.dumps(info, sort_keys=True).encode())
+    return info
+
+
+def op_db_list(args):
+    """Dumps this tool made; it never deletes one, so `over_keep` names those past the last KEEP_DUMPS for an operator."""
+    arity(args, 1, 1, "db-list <stack>")
+    database(args[0])
+    rows = []
+    for path in dumps_of(args[0]):
+        manifest = path / "manifest.json"
+        rows.append(json.loads(manifest.read_text(encoding="utf-8")) if manifest.is_file() else {"stamp": path.name, "manifest": "missing"})
+    stamps = [r.get("stamp") for r in rows]
+    emit({"stack": args[0], "dumps": rows, "keep": KEEP_DUMPS, "over_keep": stamps[:-KEEP_DUMPS] if len(stamps) > KEEP_DUMPS else []})
+
+
+def op_db_dump(args):
+    arity(args, 1, 1, "db-dump <stack>")
+    emit(dump(args[0]))
+
+
+def container_running(name):
+    result = run([DOCKER, "inspect", "--format", "{{.State.Running}}", name], 30)
+    return result.returncode == 0 and result.stdout.strip() == b"true"
+
+
+def load_dump(db, path):
+    """Replays one checked dump into the stack's database; psql stops at the first error."""
+    with gzip.open(path, "rb") as source:
+        process = subprocess.Popen([DOCKER, "exec", "-i", db["db"], "psql", "-U", db["user"], "-d", db["database"],
+                                    "-X", "-q", "-v", "ON_ERROR_STOP=1"],
+                                   stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        shutil.copyfileobj(source, process.stdin)
+        process.stdin.close()
+        code = process.wait(timeout=3600)
+    if code != 0:
+        raise Failed(f"psql restore exited {code}; its output was withheld")
+
+
+def op_db_restore(args):
+    """Restores one of this tool's dumps into an empty database, such as a new one after a major-version change.
+
+    It refuses a database that holds any table, so it never replaces data;
+    emptying a database first stays an operator decision.
+    """
+    usage = "db-restore <stack> <stamp>"
+    arity(args, 2, 2, usage)
+    db = database(args[0])
+    stamp = need(STAMP, args[1], usage)
+    source = DUMPS / args[0] / stamp / f"{args[0]}.sql.gz"
+    if not source.is_file() or source.is_symlink():
+        raise Failed(f"no dump {stamp} for {args[0]}")
+    recorded = (source.parent / "SHA256SUMS").read_text(encoding="utf-8").split()[0]
+    if sha256_file(source) != recorded:
+        raise Failed("the dump does not match its checksum; nothing was changed")
+    before = counts(db)
+    if before["public_tables"] != 0:
+        raise Failed(f"{db['database']} holds {before['public_tables']} tables; db-restore only fills an empty database")
+    app_was_running = container_running(db["app"])
+    if app_was_running:
+        must([DOCKER, "stop", db["app"]], 120, f"docker stop {db['app']}")
+    outcome = {"stack": args[0], "restored": stamp, "before": before}
+    try:
+        load_dump(db, source)
+        outcome["after"] = counts(db)
+        outcome["ok"] = True
+    except Failed as error:
+        outcome["ok"] = False
+        outcome["error"] = f"{error}; the database may hold part of the dump"
+    finally:
+        if app_was_running:
+            run([DOCKER, "start", db["app"]], 120)
+            outcome["app_started"] = container_running(db["app"])
+    emit(outcome)
+    if not outcome["ok"]:
+        raise Failed("restore failed; see the JSON above")
+
+
+COMMANDS = {
+    "container-memory": op_container_memory,
+    "container-stats": op_container_stats,
+    "container-logs": op_container_logs,
+    "container-oom": op_container_oom,
+    "kernel-oom": op_kernel_oom,
+    "host-audit": op_host_audit,
+    "host-capacity": op_host_capacity,
+    "host-storage": op_host_storage,
+    "docker-usage": op_docker_usage,
+    "service-check": op_service_check,
+    "crawl4ai-info": fixed_probe("host_probe", "crawl4ai_info"),
+    "recyclarr-naming": fixed_probe("host_probe", "recyclarr_naming"),
+    "arcane-version": fixed_probe("host_probe", "arcane_version"),
+    "queue-callback": fixed_probe("host_probe", "queue_callback"),
+    "bazarr-providers": fixed_probe("incident_probe", "bazarr_providers"),
+    "mealie-api": op_mealie_api,
+    "radarr-interface": op_radarr_interface,
+    "radarr-movie": op_radarr_movie,
+    "queue-job": op_queue_job,
+    "object-stat": op_object_stat,
+    "manager-state": op_manager_state,
+    "manager-prepare": op_manager_prepare,
+    "manager-pull": op_manager_pull,
+    "manager-backup": op_manager_backup,
+    "manager-start": op_manager_start,
+    "manager-verify": op_manager_verify,
+    "manager-rollback": op_manager_rollback,
+    "db-list": op_db_list,
+    "db-dump": op_db_dump,
+    "db-restore": op_db_restore,
+}
+
+
+def main(argv):
+    if not argv or argv[0] not in COMMANDS:
+        print("refused: unknown operation; known: " + " ".join(sorted(COMMANDS)), file=sys.stderr)
+        return 2
+    try:
+        COMMANDS[argv[0]](argv[1:])
+    except Refused as error:
+        print(f"refused: {error}", file=sys.stderr)
+        return 2
+    except Failed as error:
+        print(f"failed: {error}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    os.umask(0o077)
+    sys.exit(main(sys.argv[1:]))