@microck/raft 0.0.0-stage → 0.2.0

package/deploy/verify-fresh-host.py

@@ -0,0 +1,301 @@
+"""Provision and reboot a disposable KVM Ubuntu host using the canonical deployer.
+
+CI-only harness. Writes temporary verified SSH/configuration in its own HOME.
+Uses a sparse 100 GiB virtual disk, not an 80 GiB physical capacity guarantee.
+"""
+
+import argparse
+import hashlib
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import time
+import urllib.request
+
+ROOT = Path(__file__).resolve().parent.parent
+USER_AGENT = "OpenAI File Downloader, XaiImageApiFetch/1.0"
+BASE = "https://cloud-images.ubuntu.com/noble/current/"
+
+
+def run(*args, **kwargs):
+    return subprocess.run(args, check=True, **kwargs)
+
+
+def verify_firewall(env):
+    """Disrupt only this harness's disposable VM to prove the startup dependency."""
+    ssh = ["ssh", "raft-fresh"]
+    requires = subprocess.check_output(
+        [*ssh, "systemctl show incus.service -p Requires -p After"], env=env, text=True
+    )
+    assert all(
+        "raft-network.service" in line.split("=", 1)[1].split() for line in requires.splitlines()
+    )
+    started = subprocess.check_output(
+        [*ssh, "systemctl show incus.service -p ExecMainStartTimestampMonotonic --value"],
+        env=env,
+        text=True,
+    )
+    # Deliberately inject an obsolete allowance with no boxes present, then redeploy.
+    run(
+        *ssh,
+        "sudo iptables -N RAFT-VERIFY-UNRELATED && "
+        "sudo iptables -A RAFT-VERIFY-UNRELATED -j RETURN && "
+        "sudo iptables -I RAFT-FORWARD 1 -d 169.254.0.0/16 -j ACCEPT",
+        env=env,
+    )
+    run(sys.executable, str(ROOT / "deploy/deploy.py"), "--location", "lab", env=env)
+    run(
+        *ssh,
+        "sudo iptables -C RAFT-FORWARD -d 169.254.0.0/16 -j REJECT && "
+        "! sudo iptables -C RAFT-FORWARD -d 169.254.0.0/16 -j ACCEPT && "
+        "sudo iptables -C RAFT-VERIFY-UNRELATED -j RETURN && "
+        "sudo iptables -F RAFT-VERIFY-UNRELATED && sudo iptables -X RAFT-VERIFY-UNRELATED",
+        env=env,
+    )
+    unchanged = subprocess.check_output(
+        [*ssh, "systemctl show incus.service -p ExecMainStartTimestampMonotonic --value"],
+        env=env,
+        text=True,
+    )
+    assert started == unchanged, "Firewall reload restarted the Incus daemon"
+    run(*ssh, "sudo cp /usr/local/lib/raft/network /tmp/raft-network-proof", env=env)
+    try:
+        run(
+            *ssh,
+            "sudo systemctl stop raft-expire.timer incus-startup.service incus.socket incus.service raft-network.service",
+            env=env,
+        )
+        run(
+            *ssh,
+            "printf '#!/bin/sh\\nexit 1\\n' | sudo tee /usr/local/lib/raft/network >/dev/null",
+            env=env,
+        )
+        failed = subprocess.run(
+            [*ssh, "sudo systemctl start incus.service"], env=env, capture_output=True, text=True
+        )
+        assert failed.returncode != 0, "Incus started despite a failed firewall dependency"
+        active = subprocess.run(
+            [*ssh, "systemctl is-active incus.service"], env=env, capture_output=True, text=True
+        )
+        assert active.stdout.strip() != "active"
+        run(*ssh, "sudo systemctl start incus.socket", env=env)
+        failed = subprocess.run(
+            [*ssh, "sudo timeout 20 incus query /1.0"], env=env, capture_output=True, text=True
+        )
+        assert failed.returncode != 0, "Socket activation bypassed the firewall dependency"
+        active = subprocess.run(
+            [*ssh, "systemctl is-active incus.service"], env=env, capture_output=True, text=True
+        )
+        assert active.stdout.strip() != "active"
+    finally:
+        run(
+            *ssh,
+            "sudo cp /tmp/raft-network-proof /usr/local/lib/raft/network && sudo rm /tmp/raft-network-proof && "
+            "sudo systemctl reset-failed incus.service raft-network.service && "
+            "sudo systemctl reload-or-restart raft-network.service && "
+            "sudo systemctl start incus.socket incus.service incus-startup.service raft-expire.timer",
+            env=env,
+        )
+    print(
+        "Repeat deploy removed stale rules without restarting Incus; firewall failure blocked normal and socket startup",
+        flush=True,
+    )
+
+
+def verify(archive):
+    # Fail rather than silently switching to software emulation.
+    if not os.access("/dev/kvm", os.R_OK | os.W_OK):
+        raise RuntimeError("Fresh-host verification requires accessible KVM")
+    with tempfile.TemporaryDirectory(prefix="raft-fresh-") as directory:
+        work = Path(directory)
+        home = work / "controller"
+        ssh = home / ".ssh"
+        config = home / ".config/raft"
+        ssh.mkdir(parents=True, mode=0o700)
+        config.mkdir(parents=True, mode=0o700)
+        for key in ["client", "host"]:
+            run("ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", str(work / key))
+        (ssh / "config").write_text(
+            f"Host raft-fresh\n  HostName localhost\n  Port 2222\n  User raftci\n"
+            f"  IdentityFile {work / 'client'}\n  IdentitiesOnly yes\n"
+            f"  UserKnownHostsFile {ssh / 'known_hosts'}\n  StrictHostKeyChecking yes\n"
+        )
+        (ssh / "known_hosts").write_text("[localhost]:2222 " + (work / "host.pub").read_text())
+        configuration = config / "incus.json"
+        configuration.write_text(json.dumps({"lab": {"ssh": "raft-fresh", "image": ""}}))
+        configuration.chmod(0o600)
+        # A per-harness HOME keeps the operator's controller config untouched.
+        env = {**os.environ, "HOME": str(home)}
+        cloud = {
+            "hostname": "raft-fresh",
+            "users": [
+                {
+                    "name": "raftci",
+                    "sudo": "ALL=(ALL) NOPASSWD:ALL",
+                    "shell": "/bin/bash",
+                    "ssh_authorized_keys": [(work / "client.pub").read_text().strip()],
+                }
+            ],
+            "ssh_pwauth": False,
+            "ssh_keys": {
+                "ed25519_private": (work / "host").read_text(),
+                "ed25519_public": (work / "host.pub").read_text(),
+            },
+            "packages": ["curl", "gnupg", "btrfs-progs"],
+        }
+        (work / "user-data").write_text("#cloud-config\n" + json.dumps(cloud))
+        (work / "meta-data").write_text("instance-id: raft-fresh\nlocal-hostname: raft-fresh\n")
+        image = work / "noble-server-cloudimg-amd64.img"
+        request = urllib.request.Request(BASE + "SHA256SUMS", headers={"User-Agent": USER_AGENT})
+        sums = urllib.request.urlopen(request, timeout=60).read().decode()
+        checksum = next(
+            line.split()[0]
+            for line in sums.splitlines()
+            if line.split()[-1].lstrip("*") == image.name
+        )
+        run("curl", "-fsSL", "-A", USER_AGENT, BASE + image.name, "-o", str(image))
+        with image.open("rb") as downloaded:
+            assert hashlib.file_digest(downloaded, "sha256").hexdigest() == checksum
+        run("qemu-img", "resize", str(image), "100G")
+        run(
+            "cloud-localds",
+            str(work / "seed.img"),
+            str(work / "user-data"),
+            str(work / "meta-data"),
+        )
+        with (work / "serial.log").open("wb") as serial:
+            vm = subprocess.Popen(
+                [
+                    "qemu-system-x86_64",
+                    "-enable-kvm",
+                    "-cpu",
+                    "host",
+                    "-smp",
+                    "2",
+                    "-m",
+                    "8192",
+                    "-display",
+                    "none",
+                    "-serial",
+                    "stdio",
+                    "-monitor",
+                    "none",
+                    "-drive",
+                    f"file={image},format=qcow2,if=virtio",
+                    "-drive",
+                    f"file={work / 'seed.img'},format=raw,if=virtio",
+                    "-netdev",
+                    "user,id=net,hostfwd=tcp:127.0.0.1:2222-:22",
+                    "-device",
+                    "virtio-net-pci,netdev=net",
+                ],
+                stdout=serial,
+                stderr=subprocess.STDOUT,
+            )
+            try:
+                # SSH expands ~ using passwd, not HOME. Pass the fixture config explicitly
+                # through a PATH wrapper shared by CLI SSH and SCP calls.
+                binaries = work / "bin"
+                binaries.mkdir()
+                for tool in ["ssh", "scp"]:
+                    (binaries / tool).write_text(
+                        f'#!/bin/sh\nexec /usr/bin/{tool} -F "{ssh / "config"}" "$@"\n'
+                    )
+                    (binaries / tool).chmod(0o755)
+                env["PATH"] = str(binaries) + os.pathsep + env["PATH"]
+                deadline = time.monotonic() + 300
+                while time.monotonic() < deadline:
+                    if vm.poll() is not None:
+                        raise RuntimeError("Fresh VM exited during boot")
+                    probe = subprocess.run(
+                        [
+                            "ssh",
+                            "-T",
+                            "-o",
+                            "BatchMode=yes",
+                            "-o",
+                            "ConnectTimeout=2",
+                            "raft-fresh",
+                            "true",
+                        ],
+                        env=env,
+                        capture_output=True,
+                    )
+                    if probe.returncode == 0:
+                        break
+                    time.sleep(2)
+                else:
+                    raise RuntimeError("Fresh VM SSH did not become ready")
+                run("ssh", "raft-fresh", "sudo cloud-init status --wait --long", env=env)
+                run(sys.executable, str(ROOT / "deploy/deploy.py"), "--location", "lab", env=env)
+                verify_firewall(env)
+                run("scp", str(archive), "raft-fresh:/tmp/raft-image.tar.gz", env=env)
+                run(
+                    "ssh",
+                    "raft-fresh",
+                    "sudo incus --project raft image import /tmp/raft-image.tar.gz --alias raft-dev",
+                    env=env,
+                )
+                images = subprocess.check_output(
+                    [
+                        "ssh",
+                        "raft-fresh",
+                        "sudo incus --project raft image list raft-dev --format json",
+                    ],
+                    env=env,
+                    text=True,
+                )
+                fingerprint = next(
+                    item["fingerprint"]
+                    for item in json.loads(images)
+                    if any(a["name"] == "raft-dev" for a in item["aliases"])
+                )
+                configuration.write_text(
+                    json.dumps({"lab": {"ssh": "raft-fresh", "image": fingerprint}})
+                )
+                run("ssh", "raft-fresh", "rm /tmp/raft-image.tar.gz", env=env)
+                run(
+                    sys.executable,
+                    str(ROOT / "deploy/verify-live.py"),
+                    "--location",
+                    "lab",
+                    "--reboot-host",
+                    env=env,
+                )
+                # Oversize only this disposable pool after all boxes have been removed.
+                run("ssh", "raft-fresh", "sudo incus storage set raft-data size=61GiB", env=env)
+                drift = subprocess.run(
+                    [sys.executable, str(ROOT / "deploy/deploy.py"), "--location", "lab"],
+                    env=env,
+                    capture_output=True,
+                    text=True,
+                )
+                assert drift.returncode != 0 and "size=60GiB" in drift.stderr
+                size = subprocess.check_output(
+                    ["ssh", "raft-fresh", "sudo incus storage get raft-data size"],
+                    env=env,
+                    text=True,
+                )
+                assert size.strip() == "61GiB", "Deployment silently resized a drifting pool"
+                print(
+                    "Fresh provisioning, firewall failure gating, redeployment, oversize rejection and real host reboot passed",
+                    flush=True,
+                )
+            finally:
+                vm.terminate()
+                vm.wait(timeout=30)
+                # Report diagnostics only on failure; never publish fixture keys.
+                if sys.exc_info()[0]:
+                    print(
+                        (work / "serial.log").read_text(errors="replace")[-10000:], file=sys.stderr
+                    )
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("archive", type=Path)
+    args = parser.parse_args()
+    verify(args.archive.resolve())

package/deploy/verify-image-tests.py

@@ -0,0 +1,139 @@
+"""Exercise the real image inspector with clean and credential-bearing tar archives."""
+
+import hashlib
+import io
+import json
+import os
+from pathlib import Path, PurePosixPath
+import subprocess
+import sys
+import tarfile
+import tempfile
+
+ROOT = Path(__file__).resolve().parent.parent
+
+
+def verify():
+    with tempfile.TemporaryDirectory(prefix="raft-image-check-") as directory:
+        work = Path(directory)
+        key = work / "key"
+        subprocess.run(["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", str(key)], check=True)
+        home = PurePosixPath("rootfs", "home", "developer")
+        root = PurePosixPath("rootfs", "root")
+        cases = [
+            ("clean", None, tarfile.REGTYPE, "", True),
+            ("user-key", home / ".ssh/id_ed25519", tarfile.REGTYPE, "", False),
+            ("root-key", root / ".ssh/id_ed25519", tarfile.REGTYPE, "", False),
+            ("user-key-link", home / ".ssh/id_ed25519", tarfile.SYMTYPE, "/etc/passwd", False),
+            ("ssh-directory-link", home / ".ssh", tarfile.SYMTYPE, "/etc", False),
+            (
+                "user-key-hardlink",
+                home / ".ssh/id_ed25519",
+                tarfile.LNKTYPE,
+                "metadata.yaml",
+                False,
+            ),
+            ("user-key-fifo", home / ".ssh/id_ed25519", tarfile.FIFOTYPE, "", False),
+            ("key-target-link", home / "alias", tarfile.SYMTYPE, ".ssh/id_ed25519", False),
+            (
+                "key-target-hardlink",
+                home / "alias",
+                tarfile.LNKTYPE,
+                str(home / ".ssh/id_ed25519"),
+                False,
+            ),
+            (
+                "host-key-link",
+                PurePosixPath("rootfs/etc/ssh/ssh_host_ed25519_key"),
+                tarfile.SYMTYPE,
+                "/etc/passwd",
+                False,
+            ),
+            ("npm-token", home / ".npmrc", tarfile.REGTYPE, "", False),
+            ("empty-ssh-directory", home / ".ssh", tarfile.DIRTYPE, "", True),
+            (
+                "package-config",
+                PurePosixPath("rootfs/usr/lib/node_modules/npm/.npmrc"),
+                tarfile.REGTYPE,
+                "",
+                True,
+            ),
+            (
+                "package-token",
+                PurePosixPath("rootfs/usr/lib/node_modules/npm/npmrc"),
+                tarfile.REGTYPE,
+                "",
+                False,
+            ),
+            (
+                "normal-link",
+                PurePosixPath("rootfs/usr/bin/example"),
+                tarfile.SYMTYPE,
+                "../lib/example",
+                True,
+            ),
+        ]
+        for label, path, kind, target, accepted in cases:
+            case = work / label
+            case.mkdir()
+            archive = case / "image.tar.gz"
+            resources = {
+                "metadata.yaml": b"architecture: aarch64\n",
+                "rootfs/etc/machine-id": b"",
+                "rootfs/opt/raft/packages.tsv": b"example\t1\n",
+                "rootfs/opt/raft/npm-packages.json": b"{}\n",
+            }
+            with tarfile.open(archive, "w:gz") as output:
+                for name, content in resources.items():
+                    member = tarfile.TarInfo(name)
+                    member.size = len(content)
+                    output.addfile(member, io.BytesIO(content))
+                if path:
+                    member = tarfile.TarInfo(str(path))
+                    member.type = kind
+                    member.linkname = target
+                    content = (
+                        key.read_bytes()
+                        if "key" in label
+                        else b"registry=https://registry.npmjs.org/\n"
+                    )
+                    if label in {"npm-token", "package-token"}:
+                        content = b"//registry.npmjs.org/:_authToken=fixture-token\n"
+                    if kind == tarfile.REGTYPE:
+                        member.size = len(content)
+                    output.addfile(member, io.BytesIO(content) if kind == tarfile.REGTYPE else None)
+            fingerprint = hashlib.sha256(archive.read_bytes()).hexdigest()
+            response = subprocess.run(
+                [
+                    sys.executable,
+                    str(ROOT / "deploy/verify-image.py"),
+                    str(archive),
+                    "--architecture",
+                    "arm64",
+                    "--fingerprint",
+                    fingerprint,
+                ],
+                capture_output=True,
+                text=True,
+                env={
+                    **os.environ,
+                    "GITHUB_SHA": "0" * 40,
+                    "GITHUB_REPOSITORY": "Microck/raft",
+                    "GITHUB_RUN_ID": "0",
+                },
+            )
+            assert (response.returncode == 0) == accepted, (label, response.stderr)
+            if accepted:
+                manifest = json.loads((case / "manifest-arm64.json").read_text())
+                assert manifest["sha256"] == fingerprint
+                assert (case / "packages-arm64.tsv").read_bytes() == resources[
+                    "rootfs/opt/raft/packages.tsv"
+                ]
+            else:
+                assert "credential" in response.stderr.lower(), (label, response.stderr)
+                assert not (case / "manifest-arm64.json").exists()
+        print(f"Image inspector: {len(cases)} real clean/credential archive cases passed")
+
+
+if __name__ == "__main__":
+    verify()

package/deploy/verify-image.py

@@ -0,0 +1,95 @@
+"""Inspect a clean unified Incus image and write its public release manifest."""
+
+import argparse
+import hashlib
+import json
+import os
+from pathlib import Path, PurePosixPath
+import re
+import tarfile
+
+
+def credential_path(path):
+    """Reject credential locations even when a tar entry is a link or special file."""
+    return (
+        ".ssh" in path.parts
+        or (path.parent.name == "ssh" and path.name.startswith("ssh_host_"))
+        or path.name in {".netrc", "incus.json"}
+        or (path.name == ".npmrc" and path.parts[:2] in {("rootfs", "root"), ("rootfs", "home")})
+    )
+
+
+def verify(archive, architecture, fingerprint):
+    if archive.stat().st_size >= 2 * 1024**3:
+        raise ValueError("Image exceeds GitHub's per-release-asset limit")
+    with archive.open("rb") as exported:
+        digest = hashlib.file_digest(exported, "sha256").hexdigest()
+    if digest != fingerprint:
+        raise ValueError("Export SHA256 does not match its immutable Incus fingerprint")
+    wanted = {
+        "rootfs/opt/raft/packages.tsv": f"packages-{architecture}.tsv",
+        "rootfs/opt/raft/npm-packages.json": f"npm-packages-{architecture}.json",
+        "metadata.yaml": None,
+        "rootfs/etc/machine-id": None,
+    }
+    found = set()
+    with tarfile.open(archive, "r|gz") as image:
+        for member in image:
+            path = PurePosixPath(member.name)
+            if path.is_absolute() or ".." in path.parts:
+                raise ValueError("Unsafe image member: " + member.name)
+            name = str(path)
+            if not member.isdir() and (
+                credential_path(path)
+                or path.parts[:2] == ("rootfs", "workspace")
+                or (
+                    (member.issym() or member.islnk())
+                    and credential_path(PurePosixPath(member.linkname))
+                )
+            ):
+                raise ValueError("Operator content or credential entry in template: " + name)
+            if not member.isfile():
+                continue
+            # npm itself ships a package-local .npmrc. Check all npm configs
+            # for credentials rather than treating every package config as a secret.
+            if path.name in {".npmrc", "npmrc"}:
+                content = image.extractfile(member).read().decode(errors="replace")
+                if re.search(
+                    r"(?mi)^[^#;\n]*(?:_authToken|_auth|_password|password)\s*=\s*\S+", content
+                ):
+                    raise ValueError("Credential setting in npm config: " + name)
+            if name in wanted:
+                content = image.extractfile(member).read()
+                found.add(name)
+                if name.endswith("machine-id") and content.strip():
+                    raise ValueError("Template has a populated machine ID")
+                if wanted[name]:
+                    (archive.parent / wanted[name]).write_bytes(content)
+    if found != set(wanted):
+        raise ValueError("Missing template resources: " + str(set(wanted) - found))
+    manifest = {
+        "architecture": architecture,
+        "filename": archive.name,
+        "sizeBytes": archive.stat().st_size,
+        "sha256": digest,
+        "incusFingerprint": fingerprint,
+        "sourceCommit": os.environ["GITHUB_SHA"],
+        "buildRun": f"https://github.com/{os.environ['GITHUB_REPOSITORY']}/actions/runs/{os.environ['GITHUB_RUN_ID']}",
+        "base": "Debian 13 default system container",
+        "isolation": "Unprivileged system container; shares the host kernel",
+        "checks": ["extended lifecycle", "31 developer tools", "same-host full-image recovery"],
+        "licensing": "Raft is MIT; bundled third-party packages retain their licenses in /usr/share/doc and upstream distributions",
+    }
+    (archive.parent / f"manifest-{architecture}.json").write_text(
+        json.dumps(manifest, indent=2) + "\n"
+    )
+    print("Clean image verified: " + archive.name)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("archive", type=Path)
+    parser.add_argument("--architecture", choices=["arm64", "amd64"], required=True)
+    parser.add_argument("--fingerprint", required=True)
+    args = parser.parse_args()
+    verify(args.archive, args.architecture, args.fingerprint)

package/deploy/verify-limits.py

@@ -0,0 +1,110 @@
+"""Check capacity boundaries and optionally real disposable box transitions."""
+
+import argparse
+import json
+from pathlib import Path
+import subprocess
+import sys
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+from raft import capacity  # noqa: E402
+
+GIB = 1024**3
+CLI = [sys.executable, str(ROOT / "raft.py")]
+
+
+def policy_checks():
+    # These are allocation fixtures, not mocked host services. Expected counts
+    # describe the documented policy independently of its arithmetic.
+    host = {
+        "cpus": 8,
+        "memoryTotalBytes": 16 * GIB,
+        "memoryAvailableBytes": 14 * GIB,
+        "hostDiskFreeBytes": 40 * GIB,
+        "poolTotalBytes": 60 * GIB,
+        "poolFreeBytes": 30 * GIB,
+    }
+    running = {"status": "Running", "expanded_config": {"limits.cpu": "2", "limits.memory": "4GiB"}}
+    stopped = {"status": "Stopped"}
+    report = capacity(host, [running, stopped])
+    assert report["savedBoxes"] == 2 and report["activeBoxes"] == 1
+    assert report["allocated"] == {"cpus": 2, "memoryBytes": 4 * GIB}
+    size = next(r for r in report["recommendations"] if r["cpu"] == 2 and r["memoryGiB"] == 4)
+    assert (size["totalRunning"], size["additionalRunning"], size["newBoxes"]) == (3, 2, 2)
+    full = capacity(host, [stopped] * 4)
+    assert full["savedSlots"] == 0
+    assert all(r["newBoxes"] == 0 for r in full["recommendations"])
+    assert any(r["additionalRunning"] > 0 for r in full["recommendations"])
+    frozen = capacity(host, [{**running, "status": "Frozen"}])
+    assert frozen["allocated"] == report["allocated"]
+    low = capacity({**host, "memoryAvailableBytes": GIB, "poolFreeBytes": GIB}, [])
+    assert all(r["additionalRunning"] == 0 for r in low["recommendations"])
+    assert len(low["warnings"]) == 2
+    assert all(r["totalRunning"] == 0 for r in capacity({**host, "cpus": 1}, [])["recommendations"])
+    assert all(
+        r["totalRunning"] == 0
+        for r in capacity({**host, "memoryTotalBytes": GIB}, [])["recommendations"]
+    )
+    large = capacity({**host, "cpus": 64, "memoryTotalBytes": 128 * GIB}, [])
+    assert large["reserve"]["memoryBytes"] == 128 * GIB // 10
+    assert all(r["totalRunning"] == 4 for r in large["recommendations"])
+    over = capacity({**host, "cpus": 2}, [running])
+    assert over["warnings"] and all(r["additionalRunning"] == 0 for r in over["recommendations"])
+    for cpu, memory, expected in [(2, 8, 1), (4, 8, 3), (4, 16, 3), (8, 32, 4)]:
+        example = capacity({**host, "cpus": cpu, "memoryTotalBytes": memory * GIB}, [])
+        default = next(
+            r for r in example["recommendations"] if r["cpu"] == 1 and r["memoryGiB"] == 2
+        )
+        assert default["totalRunning"] == expected
+    print(
+        "Capacity policy: mixed sizes, stopped/frozen boxes, low RAM/disk, reserves and saved cap passed",
+        flush=True,
+    )
+
+
+def command(*args):
+    return subprocess.check_output([*CLI, *args], text=True).strip()
+
+
+def live_checks(location):
+    def limits():
+        return json.loads(command("limits", "--location", location, "--json"))
+
+    baseline = limits()
+    if baseline["savedSlots"] == 0:
+        raise RuntimeError("Capacity E2E needs one free saved-box slot")
+    box = command("new", "--location", location, "--cpu", "1", "--memory", "1GiB", "--ttl", "600")
+    try:
+        active = limits()
+        assert active["savedBoxes"] == baseline["savedBoxes"] + 1
+        assert active["activeBoxes"] == baseline["activeBoxes"] + 1
+        assert active["allocated"]["cpus"] == baseline["allocated"]["cpus"] + 1
+        assert active["allocated"]["memoryBytes"] == baseline["allocated"]["memoryBytes"] + GIB
+        assert active["savedSlots"] == baseline["savedSlots"] - 1
+        command("stop", box)
+        stopped = limits()
+        assert stopped["savedBoxes"] == active["savedBoxes"]
+        assert stopped["activeBoxes"] == baseline["activeBoxes"]
+        assert stopped["allocated"] == baseline["allocated"]
+        command("resume", box, "--ttl", "600")
+        resumed = limits()
+        assert resumed["activeBoxes"] == active["activeBoxes"]
+        assert resumed["allocated"] == active["allocated"]
+        assert "CPU  RAM GiB" in command("limits", "--location", location)
+    finally:
+        command("destroy", box)
+    final = limits()
+    assert final["savedBoxes"] == baseline["savedBoxes"]
+    assert final["activeBoxes"] == baseline["activeBoxes"]
+    assert final["allocated"] == baseline["allocated"]
+    print(f"{location}: real capacity create/stop/resume/destroy reporting passed", flush=True)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--location", help="Also test a real host using one disposable box")
+    args = parser.parse_args()
+    policy_checks()
+    if args.location:
+        live_checks(args.location)