@microck/raft 0.3.0 → 0.5.0

package/deploy/verify-self-update.py

@@ -0,0 +1,372 @@
+"""Exercise self-update and the passive release notice without GitHub or npmjs.
+
+A local HTTP server plays the GitHub releases API, the release assets and an npm
+registry. Real npm and uv installs in temporary directories are upgraded to a
+locally built 99.0.0 release; nothing outside those directories changes.
+"""
+
+import argparse
+import base64
+import contextlib
+import hashlib
+import http.server
+import io
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+import raft  # noqa: E402
+
+CURRENT = raft.__version__
+NEXT = "99.0.0"
+# These checks own the notice setting; CI may disable it for the other suites.
+os.environ.pop("RAFT_NO_UPDATE_CHECK", None)
+
+
+class Releases(http.server.BaseHTTPRequestHandler):
+    """Serve fixed bytes per path and record every request."""
+
+    routes = {}
+    requests = []
+    delay = 0
+
+    def do_GET(self):
+        Releases.requests.append(self.path)
+        time.sleep(Releases.delay)
+        body = Releases.routes.get(self.path)
+        if body is None:
+            self.send_error(404)
+            return
+        self.send_response(200)
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+    def log_message(self, *arguments):
+        pass
+
+
+def serve_release(base, version, wheel, checksums):
+    """Publish a GitHub-style release whose SHA256SUMS text is chosen by the caller."""
+    tag = "v" + version
+    assets = {wheel.name: wheel.read_bytes(), "SHA256SUMS": checksums.encode()}
+    metadata = {
+        "tag_name": tag,
+        "html_url": f"{base}/releases/tag/{tag}",
+        "assets": [
+            {"name": name, "browser_download_url": f"{base}/assets/{tag}/{name}"} for name in assets
+        ],
+    }
+    for name, content in assets.items():
+        Releases.routes[f"/assets/{tag}/{name}"] = content
+    Releases.routes["/api/latest"] = Releases.routes[f"/api/tags/{tag}"] = json.dumps(
+        metadata
+    ).encode()
+
+
+def build(source, destination):
+    """Build the wheel and npm tarball of a source tree."""
+    subprocess.run(
+        ["uv", "build", "-q", "--wheel", "-o", str(destination), str(source)], check=True
+    )
+    subprocess.run(
+        ["npm", "pack", "--silent", "--ignore-scripts", "--pack-destination", str(destination)],
+        cwd=source,
+        check=True,
+        capture_output=True,
+    )
+    return next(destination.glob("*.whl")), next(destination.glob("*.tgz"))
+
+
+def next_release(work):
+    """Build the current sources as release NEXT."""
+    source = work / "next-source"
+    source.mkdir()
+    for name in ["raft_files.py", "pyproject.toml", "README.md", "LICENSE"]:
+        shutil.copy(ROOT / name, source)
+    code = (ROOT / "raft.py").read_text()
+    (source / "raft.py").write_text(code.replace(f'"{CURRENT}"', f'"{NEXT}"', 1))
+    metadata = json.loads((ROOT / "package.json").read_text())
+    (source / "package.json").write_text(json.dumps({**metadata, "version": NEXT}))
+    return build(source, work / "next")
+
+
+def installed_update(python, *arguments, env, check=False, version=None, api):
+    """Run self_update inside an installed copy, as its own raft.py and sys.prefix see it."""
+    program = (
+        "import argparse, sys\n"
+        "sys.path[:0] = sys.argv[4:]\n"
+        "import raft\n"
+        "sys.exit(raft.self_update(argparse.Namespace(check=sys.argv[2] == 'True', "
+        "version=sys.argv[3] or None), api=sys.argv[1]))"
+    )
+    return subprocess.run(
+        [python, "-I", "-c", program, api, str(check), version or "", *arguments],
+        env=env,
+        capture_output=True,
+        text=True,
+    )
+
+
+def version_of(executable):
+    return subprocess.check_output([str(executable), "--version"], text=True).strip()
+
+
+def verify_versions():
+    assert raft.parse_version("0.10.0") > raft.parse_version("0.9.12"), "Compared as strings"
+    assert raft.parse_version("v1.2.3") == (1, 2, 3)
+    assert raft.release_version("v1.2.3") == "1.2.3"
+    for text in ["1.2", "1.2.3-rc1", "", "v1.2.3.4", " 1.2.3", "1.2.3\n", "latest"]:
+        try:
+            raft.parse_version(text)
+        except ValueError:
+            pass
+        else:
+            raise AssertionError("Accepted release version " + repr(text))
+    print("Release version parsing and numeric comparison passed", flush=True)
+
+
+def verify_channels(work):
+    """Refuse layouts that self-update does not own, before any network access."""
+    prefix = Path(sys.prefix).resolve()
+    for source, error in [
+        (ROOT / "raft.py", "source checkout"),
+        (work / "loose/raft.py", "neither"),
+        # A project-local npm install has no PREFIX/bin/raft link to its raft.py.
+        (work / "project/node_modules/@microck/raft/raft.py", "neither"),
+        (work / "lib/node_modules/@microck/raft/raft.py", "neither"),
+    ]:
+        source.parent.mkdir(parents=True, exist_ok=True)
+        source.touch()
+        try:
+            raft.install_channel(source.resolve(), prefix)
+        except ValueError as rejection:
+            assert error in str(rejection), rejection
+        else:
+            raise AssertionError("self-update accepted " + str(source))
+    # An editable uv tool keeps raft.py outside its environment.
+    editable = work / "tools/raft"
+    editable.mkdir(parents=True)
+    (editable / "uv-receipt.toml").touch()
+    try:
+        raft.install_channel((work / "loose/raft.py").resolve(), editable.resolve())
+    except ValueError:
+        pass
+    else:
+        raise AssertionError("self-update accepted an editable uv tool")
+    print("Checkout, unknown, project-local npm and editable uv layouts are refused", flush=True)
+
+
+def verify_uv(work, base, current_wheel, next_wheel):
+    tools, binaries = work / "uv-tools", work / "uv-bin"
+    env = {**os.environ, "UV_TOOL_DIR": str(tools), "UV_TOOL_BIN_DIR": str(binaries)}
+    subprocess.run(["uv", "tool", "install", "-q", str(current_wheel)], env=env, check=True)
+    python, executable = tools / "raft/bin/python", tools / "raft/bin/raft"
+    digest = hashlib.sha256(next_wheel.read_bytes()).hexdigest()
+    for checksums, error in [
+        (f"{'0' * 64}  {next_wheel.name}\n", "does not match"),
+        (f"{digest}  raft-{NEXT}.tar.gz\n", "does not match"),
+    ]:
+        serve_release(base, NEXT, next_wheel, checksums)
+        rejected = installed_update(python, env=env, api=base + "/api")
+        assert rejected.returncode == 1 and error in rejected.stderr, rejected
+        assert version_of(executable) == CURRENT, "A failed verification changed the install"
+    serve_release(base, NEXT, next_wheel, f"{digest}  {next_wheel.name}\n")
+    # uv would install into its default tool directory; refuse instead of moving.
+    moved = {key: value for key, value in env.items() if key != "UV_TOOL_DIR"}
+    rejected = installed_update(python, env=moved, api=base + "/api")
+    assert rejected.returncode == 1 and "UV_TOOL_DIR" in rejected.stderr, rejected
+    updated = installed_update(python, env=env, api=base + "/api")
+    assert updated.returncode == 0, updated
+    assert updated.stdout.splitlines()[0] == f"raft {CURRENT} -> {NEXT}", updated.stdout
+    assert "raft doctor" in updated.stdout and "Verified" in updated.stderr, updated
+    assert version_of(executable) == NEXT and version_of(binaries / "raft") == NEXT
+    current = installed_update(python, env=env, api=base + "/api")
+    assert current.returncode == 0 and "up to date" in current.stdout, current
+    print("uv tool update, SHA256SUMS refusals and tool-directory guard passed", flush=True)
+
+
+def verify_npm(work, base, current_package, next_package):
+    prefix = work / "npm"
+    env = {
+        **os.environ,
+        "npm_config_registry": base + "/",
+        "npm_config_cache": str(work / "npm-cache"),
+        "npm_config_audit": "false",
+        "npm_config_fund": "false",
+        "npm_config_update_notifier": "false",
+    }
+    if sys.platform != "linux":
+        # The package declares os: linux; allow developer runs on other systems.
+        env["npm_config_force"] = "true"
+    subprocess.run(
+        ["npm", "install", "--global", "--prefix", str(prefix), str(current_package)],
+        env=env,
+        check=True,
+        capture_output=True,
+    )
+    content = next_package.read_bytes()
+    manifest = json.loads((ROOT / "package.json").read_text())
+    tarball = f"{base}/npm/{next_package.name}"
+    Releases.routes["/npm/" + next_package.name] = content
+    Releases.routes["/@microck%2fraft"] = json.dumps(
+        {
+            "name": manifest["name"],
+            "dist-tags": {"latest": NEXT},
+            "versions": {
+                NEXT: {
+                    **manifest,
+                    "version": NEXT,
+                    "dist": {
+                        "tarball": tarball,
+                        "shasum": hashlib.sha1(content).hexdigest(),
+                        "integrity": "sha512-"
+                        + base64.b64encode(hashlib.sha512(content).digest()).decode(),
+                    },
+                }
+            },
+        }
+    ).encode()
+    serve_release(base, NEXT, work / "next" / f"raft-{NEXT}-py3-none-any.whl", "")
+    package = prefix / "lib/node_modules/@microck/raft"
+    updated = installed_update(sys.executable, str(package), env=env, api=base + "/api")
+    assert updated.returncode == 0, updated
+    assert updated.stdout.splitlines()[0] == f"raft {CURRENT} -> {NEXT}", updated.stdout
+    assert version_of(prefix / "bin/raft") == NEXT
+    assert any(path.startswith("/@microck") for path in Releases.requests), "npm skipped registry"
+    print("npm global update through the registry passed", flush=True)
+
+
+def verify_check(base, next_wheel):
+    with tempfile.TemporaryDirectory() as cache:
+        os.environ["XDG_CACHE_HOME"] = cache
+        for latest, status, text in [
+            (NEXT, raft.UPDATE_AVAILABLE, f"raft {NEXT} is available (you have {CURRENT})"),
+            (CURRENT, 0, "up to date"),
+        ]:
+            serve_release(base, latest, next_wheel, "")
+            output = io.StringIO()
+            with contextlib.redirect_stdout(output):
+                result = raft.self_update(
+                    argparse.Namespace(check=True, version=None), api=base + "/api"
+                )
+            assert result == status and text in output.getvalue(), (result, output.getvalue())
+        try:
+            raft.self_update(argparse.Namespace(check=False, version="98.0.0"), api=base + "/api")
+        except ValueError as error:
+            # The checkout refusal comes first; it proves no download was attempted.
+            assert "source checkout" in str(error), error
+        else:
+            raise AssertionError("self-update ran from a source checkout")
+    print("self-update --check exit status passed", flush=True)
+
+
+def notice(action, api):
+    """Run one passive check and return (stderr text, seconds)."""
+    stderr = io.StringIO()
+    started = time.monotonic()
+    with contextlib.redirect_stderr(stderr):
+        raft.update_check(action, api=api)()
+    return stderr.getvalue(), time.monotonic() - started
+
+
+def verify_notice(work, base, next_wheel):
+    serve_release(base, NEXT, next_wheel, "")
+    expected = f"raft {NEXT} is available (you have {CURRENT}). Run: raft self-update\n"
+    api = base + "/api"
+    with tempfile.TemporaryDirectory() as cache:
+        os.environ["XDG_CACHE_HOME"] = cache
+        cached = Path(cache) / "raft/update.json"
+        Releases.requests.clear()
+        assert notice("list", api)[0] == expected
+        assert Releases.requests == ["/api/latest"], Releases.requests
+        # Within the interval there is neither a request nor a second notice.
+        assert notice("list", api)[0] == "" and len(Releases.requests) == 1
+        assert notice("self-update", api)[0] == ""
+        cached.write_text("not json")
+        os.environ["RAFT_NO_UPDATE_CHECK"] = "1"
+        assert notice("list", api)[0] == "" and len(Releases.requests) == 1
+        del os.environ["RAFT_NO_UPDATE_CHECK"]
+        assert notice("list", api)[0] == expected and len(Releases.requests) == 2
+        # An old check expires; an unreachable API stays silent and is not retried.
+        cached.write_text(json.dumps({"checkedAt": time.time() - 25 * 3600}))
+        assert notice("list", "http://127.0.0.1:9/api")[0] == ""
+        assert time.time() - json.loads(cached.read_text())["checkedAt"] < 60
+        assert notice("list", api)[0] == "" and len(Releases.requests) == 2
+        # A hanging API costs at most the bounded wait.
+        cached.unlink()
+        Releases.delay = 5
+        try:
+            output, seconds = notice("list", api)
+        finally:
+            Releases.delay = 0
+        assert output == "" and seconds < raft.UPDATE_CHECK_TIMEOUT + 1, (output, seconds)
+    # The CLI adds the notice to stderr only; stdout stays byte-identical.
+    home = work / "home"
+    config = home / ".config/raft/incus.json"
+    config.parent.mkdir(parents=True)
+    config.write_text(json.dumps({"lab": {"ssh": "lab-host"}}))
+    cache = home / ".cache/raft/update.json"
+    cache.parent.mkdir(parents=True)
+    env = {key: value for key, value in os.environ.items() if key != "XDG_CACHE_HOME"}
+    # main() reads RELEASES_API when it runs, so the CLI can target the local server.
+    entry = (
+        f"import sys; sys.path.insert(0, {str(ROOT)!r}); import raft; "
+        f"raft.RELEASES_API = {api!r}; sys.argv[0] = 'raft'; sys.exit(raft.main())"
+    )
+
+    def cli(*arguments, **extra):
+        return subprocess.run(
+            [sys.executable, "-c", entry, *arguments],
+            env={**env, "HOME": str(home), **extra},
+            capture_output=True,
+            text=True,
+        )
+
+    listing = cli("locations")
+    assert listing.stdout == '{"location": "lab", "default": true, "note": ""}\n', listing
+    assert listing.stderr == expected, listing
+    # The notice also follows a failed command, but only once per interval.
+    cache.unlink()
+    failed = cli("note", "missing", "x")
+    assert failed.returncode == 1 and failed.stderr.endswith(expected), failed
+    again = cli("note", "missing", "x")
+    assert again.returncode == 1 and not again.stderr.endswith(expected), again
+    for response in [
+        cli("locations", RAFT_NO_UPDATE_CHECK="1"),
+        cli("--version"),
+        cli("self-update", "--version", CURRENT),
+    ]:
+        assert response.returncode == 0 and response.stderr == "", response
+    print("Passive notice throttling, opt-out, failures and stderr-only output passed", flush=True)
+
+
+def main():
+    verify_versions()
+    server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Releases)
+    threading.Thread(target=server.serve_forever, daemon=True).start()
+    base = f"http://127.0.0.1:{server.server_address[1]}"
+    try:
+        with tempfile.TemporaryDirectory(prefix="raft-self-update-") as directory:
+            work = Path(directory).resolve()
+            verify_channels(work)
+            current_wheel, current_package = build(ROOT, work / "current")
+            next_wheel, next_package = next_release(work)
+            verify_check(base, next_wheel)
+            verify_notice(work, base, next_wheel)
+            verify_uv(work, base, current_wheel, next_wheel)
+            verify_npm(work, base, current_package, next_package)
+    finally:
+        server.shutdown()
+
+
+if __name__ == "__main__":
+    main()

package/docs/boat-capabilities.md

@@ -18,11 +18,11 @@ Tests verify Raft behavior, not hosted Boat infrastructure. See [verification](.
 | Create/list/inspect/delete | ✅ | ✅ | Qualified handles, explicit locations, native Incus metadata. Single operator; no organization/account API |
 | Stop/resume | ✅ | ✅ | Stops processes, retains disk; resume requires TTL. Shared-kernel containers, not Boat VMs |
 | Running lifetime | ✅ | ✅ | Host-enforced TTL; extend resets deadline without restart. Long locked transactions delay expiry checks |
-| Sizing | ✅ | ✅ | 1/2 CPUs and 1/2/4/8 GiB choices. Resume/fork support sizing overrides; Boat offers larger sizes |
-| Host capacity recommendations | ❌ | ✅ | Raft reports host CPU/RAM budgets, active allocations, saved slots and disk headroom. Boat limits describe hosted plan credits and concurrency, not operator-host sizing. Raft enforces four saved boxes per host |
+| Sizing | ✅ | ✅ | Any whole CPU count and GiB of RAM up to the host, plus an optional own disk. Resume/fork support CPU/RAM overrides; Boat offers fixed machine sizes |
+| Host capacity estimates | ❌ | ✅ | Raft reports advisory host CPU/RAM budgets, active allocations, headroom and disk space, and enforces no box count. Boat limits describe hosted plan credits and concurrency, not operator-host sizing |
 | Hosted credit and start-rate limits | ✅ | ❌ | Boat reports plan credits, concurrency and start rates. Raft has no billing service |
 | Resize on resume/fork | ✅ | ✅ | Optional CPU/RAM overrides, retaining omitted limits |
-| Usage | ✅ | ✅ | Cgroup memory, CPU counters, shared pool space. No billing meter or per-box disk accounting |
+| Usage | ✅ | ✅ | Cgroup memory, CPU counters, and shared-pool or own-disk space. No billing meter |
 | Root terminal | ✅ | ✅ | PTY over host SSH and native Incus exec. No directly exposed guest SSH endpoint |
 | Synchronous exec | ✅ | ✅ | Literal argv and preserved exit code. Shell syntax requires an explicit shell |
 | Detached exec | ✅ | ✅ | Guest systemd unit, status, logs, cancellation. No process checkpoint or restart after stop |
@@ -39,8 +39,8 @@ Tests verify Raft behavior, not hosted Boat infrastructure. See [verification](.
 | Desktop | ✅ | ✅ | On-demand noVNC, framebuffer, pointer and browser transport checks. See verification for desktop interaction coverage |
 | Browser-only streaming | ✅ | ❌ | Missing. Full desktop only; browser confinement deferred |
 | Network separation | ❓ | ✅ | Raft tests host/private/metadata and IPv4/IPv6 peer restrictions with startup-gated filtering. Boat's equivalent filtering policy is not established; its VM boundary is listed separately |
-| Bounded host storage pool | ❓ | ✅ | Raft configures a shared 60 GiB Btrfs host pool and rejects size drift. Boat does not document an operator-managed host storage pool |
-| Strict per-box disk allocation | ✅ | ❌ | Nested subvolumes prevent claiming strict quotas; no independent Boat-sized disks |
+| Bounded host storage pools | ❓ | ✅ | Raft configures a shared 60 GiB Btrfs pool and, when enabled, a fully allocated LVM thin pool for own box disks that it never overcommits, and rejects size drift. Boat does not document an operator-managed host storage pool |
+| Strict per-box disk allocation | ✅ | ✅ | On hosts with own box disks enabled, `new --disk SIZE` gives a box a fixed-size, copy-on-write ext4 disk that guest and Docker writes cannot exceed. Shared-pool boxes, the default, have no per-box quota |
 | VM kernel isolation | ✅ | ❌ | VM deployment not implemented |
 | Nested KVM / Android emulator acceleration | ✅ | ❌ | Boat documents KVM on standard hosts; overflow cloud VMs lack it. Raft has no guest KVM device or VM mode |
 | External guest SSH | ✅ | ❌ | Boat supports a public IP or an SSH endpoint; public IPv4 is not guaranteed. Raft uses host-mediated private access |
@@ -50,9 +50,10 @@ Tests verify Raft behavior, not hosted Boat infrastructure. See [verification](.
 | Automatic deletion/retention | ✅ | ✅ | Disposable delete-on-stop/expiry; opt-in stopped-box retention cleanup with preview |
 | Dashboard | ✅ | ❌ | Missing. CLI and private desktop cover current workflow |
 | SDK/API keys | ✅ | ❌ | Host SSH authentication. No remote Raft service API |
-| CLI-wide JSON mode | ✅ | ❌ | Boat supports JSON/JSONL for most commands; argument errors may use stderr. Raft provides JSON for list/info/usage/limits and JSONL for prune |
+| CLI-wide JSON mode | ✅ | ❌ | Boat supports JSON/JSONL for most commands; argument errors may use stderr. Raft provides JSON for list/info/usage/capacity and JSONL for prune |
 | Snapshot file browsing/download | ✅ | ❌ | Full native archives only; no Raft snapshot tree or selected-file interface |
-| CLI self-update/completions | ✅ | ❌ | Standard npm or Python package installation/help. No CLI update command or dynamic shell completions |
+| CLI self-update | ✅ | ✅ | `raft self-update` upgrades npm global installs through the registry and uv tools from a release wheel checked against `SHA256SUMS`. A daily stderr notice reports new releases. Boat also updates itself after commands; Raft never installs without the command and never changes hosts |
+| Dynamic shell completions | ✅ | ❌ | Static `--help` only |
 | Managed agents | ✅ | ❌ | Not implemented. No prompt/events/steer/interrupt/conversation API |
 
 ## Reference check

package/docs/boot-performance.mdx

@@ -96,7 +96,7 @@ Successful CI runs attach seven-day artifacts (`raft-dev-amd64` and `raft-dev-ar
 
 Versioned public images are published under [releases](https://github.com/Microck/raft/releases) after recovery and fresh-host validation; public release assets do not expire on the seven-day CI schedule. Each archive must be imported into a host of the matching CPU architecture.
 
-The CI runner uses a sparse Btrfs pool on an ephemeral VM and does not certify the 80 GiB production free-disk preflight. Pinned images retain their contents until you configure a new fingerprint.
+The CI runner uses sparse Btrfs and LVM pools on an ephemeral VM and does not certify the production free-disk preflight. Pinned images retain their contents until you configure a new fingerprint.
 
 </details>
 

package/docs/contract.md

@@ -17,14 +17,16 @@ Raft manages persistent unprivileged native-architecture Incus system containers
 - npm publishes `@microck/raft` with the same Python sources as the wheel and source archive. Its executable is `raft`; Linux and Python 3.11.8+ are required. Installation runs no scripts and downloads no workspace images.
 - `raft --version` reports the release version without reading host configuration. The Python package, npm metadata and release tag must agree before publication.
 - Releases include Python wheels, source archives, npm tarballs and both native images with checksums. The release workflow verifies installation outside the checkout and publishes npm with provenance before creating the GitHub release.
+- **Self-update**: `raft self-update [--check | --version X.Y.Z]` takes the latest version from the GitHub releases API and compares versions numerically. It upgrades only npm global installs (through the registry, at the same prefix) and uv tools (a release wheel verified against the release `SHA256SUMS`), and refuses checkouts and other layouts. It never downgrades without `--version`, never changes hosts, and prints `raft doctor` as the next step. `--check` exits `100` when a newer release exists.
+- **Release notice**: At most once per 24 hours a command checks the latest release in the background and, if it is newer, prints one stderr line after the command. It never installs, prompts, changes stdout or exit status, or waits more than 2 seconds; failures stay silent. `RAFT_NO_UPDATE_CHECK=1` disables it, and `self-update` and `--version` skip it.
 
 ## Infrastructure and deployment
 
 - **Strict infrastructure validation**: Deployment refuses conflicting storage or network names and incomplete infrastructure rather than overwriting or repairing state.
-- **Storage pool sizing**: Existing owned Btrfs pools must retain the configured 60 GiB size; drift is rejected without automatic resizing.
+- **Storage pool sizing**: The shared Btrfs pool `raft-data` must retain its configured 60 GiB size. The own-disk pool `raft-disks` is opt-in: deployment creates it only when `--disk-pool SIZE` is given, on a fresh host or on any later redeploy, as a loop-backed LVM thin pool (`lvm.use_thinpool=true`) of SIZE whose file is allocated in full on the host disk and whose thin pool uses `--discards nopassdown`. A fresh host needs 80 GiB free, plus SIZE when `--disk-pool` is given; adding the pool later needs SIZE plus 20 GiB free. A host without `raft-disks` is healthy. An existing `raft-disks` keeps its size, a different explicit `--disk-pool` is refused, and a pool with another driver or thick volumes is rejected. Deployment never resizes either pool.
 - **Firewall precedence**: Redeployment reapplies firewall rules. Incus startup requires successful firewall application before the daemon starts, including socket activation.
 - **Guided onboarding**: `raft host add LOCATION --ssh TARGET` probes the target read-only and refuses unsupported hosts, including macOS, before any change. It prints the exact plan and applies it only after an interactive yes or `--yes`; `--dry-run` stops after the plan. It runs the same deployer as `deploy/deploy.py`, imports an image only after its SHA-256 matches (release `SHA256SUMS` or the local archive digest) and its architecture matches the host, then registers the location with the image fingerprint. It finishes with `doctor` and a disposable create/exec/destroy smoke test; if either fails, it removes the new location from the configuration and leaves host changes for a rerun. It uses existing key-based SSH and never collects passwords or changes SSH, other firewall rules or IP configuration. It requires the deployment files from the npm package or a source checkout.
-- **CLI coverage**: The single-operator workflow spans lifecycle, execution, files, snapshots, jobs, tunnels, backup, and capacity inspection; see the [command reference](./reference/cli.md).
+- **CLI coverage**: The single-operator workflow spans lifecycle, execution, files, snapshots, jobs, tunnels, backup, and advisory capacity inspection; see the [command reference](./reference/cli.md).
 
 ## Configuration and lifecycle defaults
 
@@ -33,22 +35,30 @@ Raft manages persistent unprivileged native-architecture Incus system containers
 | Host identification | Configuration requires named SSH targets using letters, digits, underscores, or hyphens. |
 | Qualified handles | Every handle includes location and instance name (`<location>:<name>`). |
 | Image pinning | Workspace creation requires the immutable 64-hex image fingerprint, not an updating alias. |
-| Lifecycle defaults | Defaults to first configured location, 1 CPU, 2 GiB RAM, and 600-second lifetime. Supported choices are 1 or 2 CPUs, 1/2/4/8 GiB RAM, and 60 to 2,592,000 seconds (30 days) TTL. |
-| Admission limit | Hard ceiling of four saved boxes per host, including stopped boxes. |
+| Lifecycle defaults | Built-in defaults are the first configured location, 1 CPU, 2 GiB RAM, the shared pool, and 600-second lifetime. `~/.config/raft/config.json` can override them for `new` only; explicit flags win. Boxes from a snapshot (`new --from`) and forks keep the source's CPU, RAM and disk instead of any default, configured or built-in; each flag overrides only its own value. `--cpu` takes any positive whole number, `--memory` any positive whole GiB (`16GiB`), `--disk` a positive whole GiB or `shared`, and TTL 60 to 2,592,000 seconds (30 days). |
+| Box count and size | Raft limits neither how many boxes a host holds nor their size. CPU or RAM limits larger than the whole host (CPU count, or `MemTotal` rounded up to whole GiB) are refused under the host lock before any change, because Incus would clip them silently. Other failures come from Incus, such as too little pool space. |
 | TTL calculation | Running TTL begins after launch completes, using the host clock. Cold image preparation does not consume TTL. |
 | Host locking | Creation, lifecycle transitions, and expiry are serialized by host file lock (`/run/lock/raft-incus.lock`). Noninteractive transactions close controller stdin so commands cannot wait for piped input while holding this lock. Exec, SSH, and file-transfer paths remain unchanged. Piped input to `raft exec` is unsupported. Native Incus admin calls bypass CLI policy. |
 
-## Capacity and admission inspection
+## Locations, notes and capacity inspection
 
-`raft locations` lists configured locations as JSONL with `location`, `default` and `note` fields without contacting hosts. `raft note LOCATION TEXT` and `raft note LOCATION --clear` edit the optional `note` field in the controller configuration atomically, preserving location order and mode `0600`. Notes are advisory guidance; Raft shows them in `locations`, `limits` and on stderr from `new`, and never enforces them.
+`raft locations` lists configured locations as JSONL with `location`, `default` and `note` fields without contacting hosts. `raft note LOCATION TEXT` and `raft note LOCATION --clear` edit the optional `note` field in the controller configuration atomically, preserving location order and mode `0600`. Notes are advisory guidance; Raft shows them in `locations`, `capacity` and on stderr from `new`, and never enforces them.
 
-`raft limits [--location NAME] [--json]` inspects hosts without modifying state. It reports effective CPUs, total and available RAM, saved and active box counts, active resource limits, and disk space.
+Boxes accept an optional note with a fixed template: `purpose` (required), `owner` and `next`, all nonempty strings. `raft new --purpose TEXT [--owner TEXT] [--next TEXT]` and `raft note BOX ...` store it as JSON in the box's `user.raft.note` Incus key; `raft note BOX --clear` removes it. Each edit replaces the whole note. `raft list` reports it as `note`, or `null`, and reports each box's `disk` as `"shared"` or its own disk size. `new` always writes the key, so `new --from` never inherits a snapshot's note. Forks and recovered backups keep it unless `fork` or `recover` gets note flags, which replace the whole note. Box notes never change lifecycle behavior.
 
-- **Reserve calculations**: Recommends running boxes after reserving 1 CPU and the larger of 2 GiB or 10% host RAM.
-- **Total capacity**: Assumes an empty dedicated host, capped at four boxes.
-- **Additional running**: Subtracts active configured limits and evaluates available RAM minus host reserves. Every non-`Stopped` state (including `Frozen`) counts as active.
-- **New box capacity**: Bounded by remaining saved-box slots: `min(more running, max(0, 4 - saved boxes))`.
-- **Advisory estimates**: CPU and memory calculations are advisory planning estimates, not admission guarantees. Disk space below 5 GiB triggers diagnostic warnings. The four-saved-box admission cap is strictly enforced.
+## Controller defaults and note policies
+
+- **Controller defaults**: `~/.config/raft/config.json` holds `{"defaults": {...}}` with optional `location`, `ttl`, `cpu`, `memory` and `disk` for `new`. Precedence is explicit flag, then configured default, then built-in default. `cpu`, `memory` and `disk` apply only to boxes from the image: `new --from` keeps the snapshot's location, CPU, RAM and disk unless a flag overrides them, and `--disk shared` is refused there. A `disk` default on a host without `raft-disks`, or larger than its unreserved space, fails like `--disk` and names `defaults.disk` as the source. Other commands take no defaults. A value is valid exactly when the same text is valid for the flag. Unknown keys, bad values and malformed JSON fail `new` with the file path and key, before any host access. `raft config set/unset` validate first, write atomically under a controller lock, keep mode `0600`, print the change and can repair the key they edit. The file is separate from `incus.json`, so location names never collide with setting names.
+- **Note policy storage**: `raft policy LOCATION --require FIELDS` stores required box note fields, which must include `purpose`, in the host's Incus project key `user.raft.require`; `--clear` empties it. Each change runs under the host lock, prints the old and new fields, records UTC time, the SSH account (`SUDO_USER`) and the self-reported controller user and hostname in `user.raft.require.changed`, and logs to the host journal with tag `raft`. Redeployment leaves these keys unchanged.
+- **Note policy enforcement**: `new` (including `--from`), `fork`, `recover` and `raft note BOX` check the note the box will carry. The controller reads the policy first and fails with the missing flags and a complete corrected command before changing anything. The host repeats the check inside the transaction that writes the note, under the host lock, so a concurrent policy change cannot be missed. That transaction also holds the oversize and own-disk reservation checks; all of them run before Incus creates anything. `fork` checks the given note or, without one, the source note read under the lock. `recover` checks a given note before upload; an archived note is checked after import, and a failing import is deleted in the same transaction. `note --clear` and replacement notes that drop a required field are rejected.
+- **Policy scope**: The policy is a guardrail for agents, not an access control. Anyone with host SSH and sudo can change it or create boxes with native Incus commands, and controllers without policy support do not check it. `raft config` reports every location's policy and last change.
+
+`raft capacity [--location NAME] [--json]` inspects hosts without modifying state. Text and JSON (`notice`) lead with: "Advisory estimate only. Raft enforces no box count or size budget; create the boxes your task needs." It reports effective CPUs, total and available RAM, saved and active box counts, active resource limits, headroom, free space in the host disk and the shared pool, and for `raft-disks` its data space, reserved and used bytes and metadata use (`boxDiskPool`, `null` when own box disks are not enabled).
+
+- **Reserve calculations**: Estimates leave out 1 CPU and the larger of 2 GiB or 10% host RAM.
+- **Example shapes**: `estimates` lists shapes doubling from 1 CPU / 2 GiB at 2 GiB per CPU while they fit an empty host, with `fitEmptyHost` and `fitNow` counts. No count is capped.
+- **Headroom**: Subtracts active configured limits and keeps available RAM above the reserve. Every non-`Stopped` state (including `Frozen`) counts as active.
+- **Advisory only**: Nothing in the report blocks creation, resume or fork. Disk space below 5 GiB and `raft-disks` thin pool metadata above 80% trigger warnings; a missing `raft-disks` does not.
 
 ## Container lifecycle and execution
 
@@ -57,12 +67,12 @@ Raft manages persistent unprivileged native-architecture Incus system containers
 - **Host worker contract**: `doctor`, disposable creation/fork, and retention cleanup require the current worker protocol. `doctor` checks it without running expiry or changing boxes, and reports the failing location. A mismatch fails before executing deletion policy or creating a disposable box; deploy the current checkout explicitly. Installing a new CLI does not update host workers.
 - **Disposable lifecycle**: `new --disposable` opts a box into deletion on stop or TTL expiry, including its snapshots. Clones and recovered boxes default to persistent; fork accepts an explicit `--disposable`.
 - **Retention cleanup**: `prune --older-than DAYS` previews persistent stopped boxes with a recorded stop timestamp. `--yes` deletes matching boxes under the host lifecycle lock. Running boxes and boxes without a recorded stop time are excluded. No automatic persistent-box deletion is enabled.
-- **Resizing**: Resume and fork accept optional supported CPU/RAM overrides, preserving omitted values. Resume validates stopped state under the lock before changing sizing.
+- **Resizing**: Resume and fork accept optional CPU/RAM overrides, preserving omitted values. Resume validates stopped state under the lock before changing sizing. Neither changes a box's disk.
 - **Expiration cleanup**: The host timer checks expirations every 15 seconds, stopping expired persistent containers and deleting expired disposable containers. Expiry may be delayed by graceful shutdown, long locked transactions, and host outage until startup.
 - **Destroy**: Removes the container instance and its snapshots, releasing storage. Destroy confirms absence; SSH failure is not proof of deletion. Disposable boxes created with `new --disposable` are deleted on explicit stop or expiry. Persistent boxes still require explicit destroy for deletion.
 - **Execution transport**: Incus exec and file APIs travel over authenticated host SSH without a public Incus listener or injected credentials. Interactive SSH runs a PTY over host SSH and Incus exec.
 - **State inspection and tunnels**: `raft list` and `raft info` read native metadata without guest process counters during shutdown; tunnels use the managed NIC.
-- **Resource reporting**: `raft usage` reports running cgroup memory, CPU microseconds, CPU affinity, and shared pool space. It does not provide per-box disk accounting or billing.
+- **Resource reporting**: `raft usage` reports running cgroup memory, CPU microseconds, CPU affinity, and filesystem space with `filesystemScope`: the shared pool, or `own SIZE box disk`. It provides no billing.
 - **Recursive transfers**: `upload/download --recursive` require a running workspace and Python 3.11.8+ in the guest. They stream a tar archive and publish a new destination directory atomically without overwriting or merging. Relative links contained within the tree, regular files, directories and internal hardlinks are supported; escaping paths/links and special files are rejected. Source entries are pinned with no-follow descriptors before metadata inspection, file reads or directory descent, so pathname replacement cannot redirect access outside the selected tree. Failed transfers leave existing destinations unchanged.
 - **Directory transfer completion**: The receiver verifies the sender's exact archive byte count before publication. A truncated stream, including one ending between complete entries, is rejected without publishing a partial tree.
 - **File transfer failures**: Controller write errors return an error before replacing a download destination or publishing a backup archive.
@@ -77,10 +87,11 @@ Raft manages persistent unprivileged native-architecture Incus system containers
 
 ## Storage, snapshots, and cloning
 
-- **Storage pool**: A dedicated 60 GiB Btrfs pool per host supports snapshots and clones. Nested container subvolumes prevent strict per-box quota enforcement. The native compressed image cache resides outside the shared pool, so host disk space still matters.
+- **Shared pool**: Boxes default to a dedicated 60 GiB Btrfs pool per host with instant snapshots and clones. Btrfs quotas cannot bound a box there, because nested subvolumes (Docker's, for example) escape them. The native compressed image cache resides outside both pools, so host disk space still matters.
+- **Own box disks**: `new --disk SIZE` puts the box's root on an LVM thin volume of SIZE in `raft-disks`, formatted ext4. All guest writes, including Docker data, land in that filesystem, so writes fail with ENOSPC at SIZE. New own-disk boxes are thin snapshots of the cached image volume, and forks, templates and snapshots of own-disk boxes are thin snapshots of their source. The pool is never overcommitted: under the host lock, before `new`, `new --from`, `fork`, `snapshot` or `recover` adds thin volumes to `raft-disks`, Raft sums the sizes of every thin volume in the pool (from `lvs`) plus the new ones and refuses, before creating anything, if the total exceeds the thin pool's data space. A new volume counts as the larger of its size and its source's; the first own-disk box on a host also counts the image volume (`volume.size`, or 10 GiB); a recovered backup counts its size once per snapshot plus once. `restore` reserves nothing. On a host without `raft-disks`, `new --disk` and recovery of an own-disk backup fail before creating anything and name the deploy command that enables the pool. `fork` and `new --from` keep the source disk; `new --from ... --disk SIZE` copies onto an own disk of SIZE. A disk too small for its contents fails creation.
 - **Stopped-state requirement**: Snapshots, restores and forks require a stopped source for consistent disk contents. Operations run under the host lifecycle lock.
 - **Snapshot restore**: `raft restore` rolls back disk blocks via `--diskonly`, retaining container configuration, limits, and MAC address.
-- **Snapshot management**: `snapshot-delete <box> <name>` removes one named snapshot under the lock. `new --from LOCATION:BOX/SNAPSHOT` creates a same-host box from that snapshot, with normal creation defaults and a fresh network identity. The source may run; the named snapshot is immutable. Templates retain filesystem secrets.
+- **Snapshot management**: `snapshot-delete <box> <name>` removes one named snapshot under the lock. `new --from LOCATION:BOX/SNAPSHOT` creates a same-host box from that snapshot, with the snapshot's CPU, RAM and disk unless `--cpu`, `--memory` or `--disk` overrides them, and a fresh network identity. The source may run; the named snapshot is immutable. Templates retain filesystem secrets.
 - **Workspace forks**: `raft fork` copies filesystem data and secrets to a new container with an independent Incus name, fresh MAC address, and renewed TTL. Source snapshots are not inherited.
 
 ## Background jobs and desktop
@@ -92,5 +103,5 @@ Raft manages persistent unprivileged native-architecture Incus system containers
 
 - **Export safeguards**: `raft backup` exports stopped containers and snapshots to a local archive. Destination path must not exist, requires mode `0600`, and uses atomic hard-link publication.
 - **Host locking**: Export holds the host lifecycle lock, temporarily delaying expiry checks on the source host.
-- **Recovery admission**: `raft recover` imports archives into a stopped container with a fresh name, new MAC address, and cleared TTL, while enforcing the four-box admission limit. Recovery never overwrites existing boxes.
+- **Recovery**: `raft recover` imports archives into a stopped container with a fresh name, new MAC address, and cleared TTL. It reads the archive's recorded pool from `backup/index.yaml`: own-disk backups return to `raft-disks` with their size after the reservation check, failing on a host without the pool, and other archives land in `raft-data`. The host note policy applies after the reservation check. Recovery never overwrites existing boxes.
 - **Archive security**: Archives contain full container filesystems, including credentials. Native archive configuration is not sanitized; backups do not imply scheduled off-host uploads. Store archives privately and import only trusted exports.

package/docs/guides/backup-recovery.mdx

@@ -54,7 +54,7 @@ echo "Recovered workspace: $recovered"
 - **Architecture compatibility**: Target host must match the native CPU architecture (ARM64 to ARM64, AMD64 to AMD64). Cross-architecture recovery is not supported.
 - **Fresh network identity**: The container receives a new name in project `raft`, a new MAC address on `rfbr0`, and no inherited TTL deadline.
 - **Stopped initial state**: Recovered workspaces remain stopped until started with `raft resume "$recovered" --ttl <SECONDS>`.
-- **Enforced admission limit**: The destination location must hold fewer than four boxes.
+- **Disk placement**: A backup of a box with its own disk returns to an own disk of the same size in `raft-disks`. The destination needs own box disks enabled and enough unreserved pool space for the box and each of its snapshots; otherwise recovery fails before importing. Other backups land in the shared pool.
 - **Interrupted recovery**: Follow [Host crash during recovery](./troubleshooting.mdx#host-crash-during-recovery). Inspect the reported handle and container state before deleting any staging file.
 
 Start and verify the recovered workspace: