@microck/raft 0.5.0 → 0.6.0

package/deploy/verify-features.py

@@ -28,6 +28,8 @@ from raft import (  # noqa: E402
     CHECK_NOTE_CODE,
     READ_NOTE_POLICY,
     RESERVE_BOX_DISK,
+    RETENTION_KEY,
+    RETENTION_SINCE_KEY,
     disk_origin,
     host_problems,
     incus,
@@ -265,7 +267,7 @@ def verify_controller_defaults():
                 "import json, sys; import raft; args = raft.parser().parse_args(sys.argv[1:]); "
                 "used = raft.apply_new_defaults(args); "
                 "print(json.dumps([args.location, args.ttl, args.cpu, args.memory, args.disk, "
-                "sorted(used)]))"
+                "args.disposable, sorted(used)]))"
             )
             response = subprocess.run(
                 [sys.executable, "-c", program, *arguments],
@@ -279,13 +281,14 @@ def verify_controller_defaults():
             return json.loads(response.stdout)
 
         # No disk is the shared pool.
-        assert resolved("new") == ["lab", 600, 1, "2GiB", None, []]
+        assert resolved("new") == ["lab", 600, 1, "2GiB", None, False, []]
         for arguments, printed in [
             (["set", "cpu", "2"], "defaults.cpu: built-in default -> 2"),
             (["set", "memory", "4GiB"], "defaults.memory: built-in default -> 4GiB"),
             (["set", "ttl", "3600"], "defaults.ttl: built-in default -> 3600"),
             (["set", "location", "spare"], "defaults.location: built-in default -> spare"),
             (["set", "disk", "20GiB"], "defaults.disk: built-in default -> 20GiB"),
+            (["set", "disposable", "true"], "defaults.disposable: built-in default -> true"),
             (["set", "cpu", "1"], "defaults.cpu: 2 -> 1"),
         ]:
             response = cli("config", *arguments)
@@ -297,29 +300,48 @@ def verify_controller_defaults():
                 "ttl": 3600,
                 "location": "spare",
                 "disk": "20GiB",
+                "disposable": True,
             }
         }
         assert config.stat().st_mode & 0o777 == 0o600, "Default editing widened permissions"
         # Explicit flags win over config.json, which wins over built-in defaults.
-        everything = ["cpu", "disk", "location", "memory", "ttl"]
-        assert resolved("new") == ["spare", 3600, 1, "4GiB", "20GiB", everything]
+        everything = ["cpu", "disk", "disposable", "location", "memory", "ttl"]
+        assert resolved("new") == ["spare", 3600, 1, "4GiB", "20GiB", True, everything]
         assert resolved("new", "--location", "lab", "--ttl", "60", "--cpu", "2") == [
             "lab",
             60,
             2,
             "4GiB",
             "20GiB",
-            ["disk", "memory"],
+            True,
+            ["disk", "disposable", "memory"],
         ]
-        # --disk shared beats a configured own disk.
-        assert resolved("new", "--disk", "shared")[4:] == [
+        # --disk shared beats a configured own disk, and --persistent a disposable default.
+        assert resolved("new", "--disk", "shared", "--persistent")[4:] == [
             None,
+            False,
             ["cpu", "location", "memory", "ttl"],
         ]
+        assert resolved("new", "--disposable")[5:] == [
+            True,
+            ["cpu", "disk", "location", "memory", "ttl"],
+        ]
+        rejected = cli("new", "--disposable", "--persistent")
+        assert rejected.returncode == 2 and "not allowed with" in rejected.stderr, rejected
         # A box from a snapshot keeps the snapshot's location, CPU, RAM and disk:
         # defaults, configured or built-in, never override them; flags still do.
+        # A snapshot has no lifecycle, so the TTL and disposable defaults apply.
         template = "lab:rf-0123456789abcdef/snap"
-        assert resolved("new", "--from", template) == [None, 3600, None, None, None, ["ttl"]]
+        assert resolved("new", "--from", template) == [
+            None,
+            3600,
+            None,
+            None,
+            None,
+            True,
+            ["disposable", "ttl"],
+        ]
+        assert resolved("new", "--from", template, "--persistent")[5] is False
         assert resolved("new", "--from", template, "--cpu", "4", "--disk", "30GiB")[2:5] == [
             4,
             None,
@@ -328,24 +350,30 @@ def verify_controller_defaults():
         assert "--from keeps the snapshot's disk" in resolved(
             "new", "--from", template, "--disk", "shared"
         )
-        # fork has no defaults at all: omitted sizes stay the source's.
+        # fork has no defaults at all: omitted sizes stay the source's, and a fork
+        # is persistent without --disposable even under a disposable default.
         forked = subprocess.check_output(
             [
                 sys.executable,
                 "-c",
                 "import raft; args = raft.parser().parse_args(['fork', 'lab:rf-0123456789abcdef', "
-                "'--ttl', '600']); print(raft.sizing(args), hasattr(args, 'disk'))",
+                "'--ttl', '600']); print(raft.sizing(args), hasattr(args, 'disk'), args.disposable)",
             ],
             cwd=ROOT,
             env={**os.environ, "HOME": home},
             text=True,
         )
-        assert forked == "[] False\n", forked
+        assert forked == "[] False False\n", forked
         for arguments, error in [
             (["set", "cpu", "0"], "defaults.cpu='0' is invalid"),
             (["set", "memory", "3GB"], "defaults.memory='3GB' is invalid"),
             (["set", "disk", "20"], "defaults.disk='20' is invalid"),
             (["set", "ttl", "59"], "TTL must be 60 seconds to 30 days"),
+            (
+                ["set", "disposable", "yes"],
+                "defaults.disposable='yes' is invalid: Use true or false",
+            ),
+            (["set", "disposable", "True"], "defaults.disposable='True' is invalid"),
             (["set", "location", "missing"], "Unknown location"),
             (["set", "location", "bad name"], "Location names"),
         ]:
@@ -359,6 +387,7 @@ def verify_controller_defaults():
             ({"defaults": {"cpu": 0}}, "defaults.cpu=0 is invalid"),
             ({"defaults": {"cpu": True}}, "defaults.cpu=True is invalid"),
             ({"defaults": {"disk": 20}}, "defaults.disk=20 is invalid"),
+            ({"defaults": {"disposable": 1}}, "defaults.disposable=1 is invalid"),
             ({"defaults": {"colour": "red"}}, "unknown key defaults.colour"),
             ({"defaults": []}, '{"defaults"'),
             ({"lab": {"ssh": "x"}}, '{"defaults"'),
@@ -389,6 +418,7 @@ def verify_controller_defaults():
                 ("ttl", "120"),
                 ("location", "lab"),
                 ("disk", "shared"),
+                ("disposable", "false"),
             ]
         ]
         assert all(writer.wait() == 0 for writer in writers)
@@ -398,8 +428,25 @@ def verify_controller_defaults():
             "ttl": 120,
             "location": "lab",
             "disk": "shared",
+            "disposable": False,
         }
-        for key in ["cpu", "memory", "ttl", "location", "disk"]:
+        # JSON booleans read back as the CLI's true and false.
+        assert cli("config", "set", "disposable", "true").stdout == (
+            "defaults.disposable: false -> true\n"
+        )
+        # raft config reports it with its source; the report's host half needs SSH.
+        effective = subprocess.check_output(
+            [
+                sys.executable,
+                "-c",
+                "import json, raft; print(json.dumps(raft.effective_defaults()['disposable']))",
+            ],
+            cwd=ROOT,
+            env={**os.environ, "HOME": home},
+            text=True,
+        )
+        assert json.loads(effective) == {"value": True, "source": "config"}, effective
+        for key in ["cpu", "memory", "ttl", "location", "disk", "disposable"]:
             assert cli("config", "unset", key).returncode == 0
         assert not config.exists(), "Removing every default left an empty file"
         assert not list(directory.glob(".raft-*")), "Staged configuration left behind"
@@ -409,13 +456,21 @@ def verify_controller_defaults():
             (["policy", "lab", "--require", "owner"], "always need purpose"),
             (["policy", "lab", "--require", "purpose,colour"], "Give note fields from"),
             (["policy", "lab", "--require", ","], "Give note fields from"),
-            (["policy", "missing", "--clear"], "Unknown location"),
+            (["policy", "missing", "--require", "none"], "Unknown location"),
+            (["policy", "lab"], "Give --require FIELDS|none, --delete-stopped-after"),
+            (["policy", "lab", "--delete-stopped-after", "0"], "positive whole number of hours"),
+            (["policy", "lab", "--delete-stopped-after", "48h"], "positive whole number of hours"),
+            # Both options are parsed before either changes the host.
+            (
+                ["policy", "lab", "--require", "none", "--delete-stopped-after", "-1"],
+                "positive whole number of hours",
+            ),
             (["fork", handle, "--ttl", "600", "--owner", "agent"], "need --purpose"),
             (["recover", "archive", "--location", "lab", "--next", "x"], "need --purpose"),
         ]:
             rejected = cli(*arguments)
             assert rejected.returncode == 1 and error in rejected.stderr, (arguments, rejected)
-        assert cli("policy", "lab").returncode == 2
+        assert cli("policy", "lab", "--clear").returncode == 2
     # The controller message names the missing flags and a complete command.
     program = (
         "import raft; print(raft.note_problem('lab', ['purpose', 'owner'], "
@@ -467,6 +522,115 @@ def verify_controller_defaults():
     print("Controller defaults, precedence, validation and note policy messages passed", flush=True)
 
 
+# Stands in for incus and logger on PATH for the real host worker. It answers the
+# worker's two reads from the state file and appends every call to the log file.
+FAKE_HOST_TOOL = """import json, os, sys
+from pathlib import Path
+arguments = [Path(sys.argv[0]).name, *sys.argv[1:]]
+with open(os.environ["RAFT_FAKE_LOG"], "a") as log:
+    log.write(json.dumps(arguments) + "\\n")
+state = json.loads(Path(os.environ["RAFT_FAKE_STATE"]).read_text())
+if arguments[1:] == ["--project", "raft", "list", "--fast", "--format", "json"]:
+    print(json.dumps(state["instances"]))
+elif arguments[1:] == ["--project", "raft", "project", "list", "--format", "json"]:
+    print(json.dumps([{"name": "default", "config": {}}, {"name": "raft", "config": state["project"]}]))
+"""
+
+
+def verify_retention_worker():
+    """Run the real host worker against fake incus fixtures and check what it changes."""
+    now = int(time.time())
+
+    def box(name, status, stopped_at=None, raft="true"):
+        config = {"user.raft": raft}
+        if stopped_at is not None:
+            config["user.raft.stopped-at"] = str(stopped_at)
+        return {"name": name, "status": status, "config": config}
+
+    instances = [
+        # Running boxes are never deleted; an old stop time left behind is cleared.
+        box("rf-running", "Running", now - 30 * 86400),
+        box("rf-old", "Stopped", now - 3 * 3600),
+        box("rf-recent", "Stopped", now - 3600),
+        box("rf-unrecorded", "Stopped"),
+        box("unrelated", "Stopped", now - 30 * 86400, raft="false"),
+    ]
+
+    def run_worker(project, *arguments):
+        with tempfile.TemporaryDirectory() as work:
+            root = Path(work)
+            for tool in ["incus", "logger"]:
+                (root / tool).write_text(f"#!{sys.executable}\n" + FAKE_HOST_TOOL)
+                (root / tool).chmod(0o755)
+            (root / "state.json").write_text(
+                json.dumps({"instances": instances, "project": project})
+            )
+            started = int(time.time())
+            response = subprocess.run(
+                [sys.executable, "-I", str(ROOT / "deploy/raft-expire.py"), *arguments],
+                env={
+                    **os.environ,
+                    "PATH": f"{root}:{os.environ['PATH']}",
+                    "RAFT_FAKE_STATE": str(root / "state.json"),
+                    "RAFT_FAKE_LOG": str(root / "calls.jsonl"),
+                },
+                capture_output=True,
+                text=True,
+            )
+            calls = [json.loads(line) for line in (root / "calls.jsonl").read_text().splitlines()]
+        # Leave out the two reads and the stale stop time cleared on the running box;
+        # normalize the recorded stop time to "now".
+        clear = ["incus", "--project", "raft", "config", "unset", "rf-running"]
+        cleared.append(clear + ["user.raft.stopped-at"] in calls)
+        changes = []
+        for call in calls:
+            if call[-2:] == ["--format", "json"] or call == clear + ["user.raft.stopped-at"]:
+                continue
+            if call[-1].startswith("user.raft.stopped-at="):
+                assert started <= int(call[-1].partition("=")[2]) <= int(time.time()), call
+                call[-1] = "user.raft.stopped-at=now"
+            changes.append(call)
+        return response, changes
+
+    cleared = []
+    record = ["incus", "--project", "raft", "config", "set", "rf-unrecorded"]
+    record.append("user.raft.stopped-at=now")
+    two_hours = {"user.raft.delete-stopped-after": "2", "user.raft.delete-stopped-after.since": ""}
+    # A 2-hour policy set long ago deletes the box stopped 3 hours ago and logs it.
+    response, changes = run_worker({**two_hours, "user.raft.delete-stopped-after.since": "0"})
+    assert response.returncode == 0, response
+    assert changes[0] == ["incus", "--project", "raft", "delete", "rf-old"], changes
+    assert changes[1][:3] == ["logger", "-t", "raft"] and "rf-old" in changes[1][3], changes
+    assert "under the 2-hour retention" in changes[1][3] and changes[2:] == [record], changes
+    assert cleared[-1], "the worker kept a stale stop time on a running box"
+    assert changes[1][3] in response.stdout, response
+    # Set an hour ago, the policy's clock starts then: nothing is deleted yet.
+    since = str(now - 3600)
+    response, changes = run_worker({**two_hours, "user.raft.delete-stopped-after.since": since})
+    assert response.returncode == 0 and changes == [record], (response, changes)
+    # Without a policy nothing is deleted, but stop times are still recorded.
+    response, changes = run_worker({})
+    assert response.returncode == 0 and changes == [record], (response, changes)
+    # Prune mode only selects; it records and retains nothing else.
+    response, changes = run_worker(
+        {**two_hours, "user.raft.delete-stopped-after.since": "0"}, "--prune-days", "1"
+    )
+    assert response.returncode == 0 and changes == [], (response, changes)
+    # An invalid policy deletes nothing, still enforces deadlines, and fails the run.
+    instances.append(
+        {
+            "name": "rf-expired",
+            "status": "Running",
+            "config": {"user.raft": "true", "user.raft.expires": "1"},
+        }
+    )
+    response, changes = run_worker({**two_hours, "user.raft.delete-stopped-after": "48h"})
+    assert response.returncode == 1 and "Invalid retention" in response.stderr, response
+    assert ["incus", "--project", "raft", "stop", "rf-expired", "--timeout", "30"] in changes
+    assert not any("delete" in call for call in changes), changes
+    print("Host worker retention, stop-time recording and grace period passed", flush=True)
+
+
 def verify_host_preflight():
     """Check onboarding decisions and refusals that need no host."""
     ready = {
@@ -588,7 +752,7 @@ def verify_note_policy(location, create, listed_note, fixtures):
         assert changed.stdout == f"{location}: required note fields none -> purpose,owner\n"
         report = json.loads(live.raft("config").stdout)
         assert report["policy"][location]["require"] == ["purpose", "owner"], report
-        assert " by " in report["policy"][location]["changed"], report
+        assert " by " in report["policy"][location]["requireChanged"], report
         saved = len(inventory(location))
         for arguments, error in [
             (["new", "--location", location, "--purpose", "x"], "requires --owner (host policy)"),
@@ -632,7 +796,7 @@ def verify_note_policy(location, create, listed_note, fixtures):
         live.raft("note", source, "--purpose", "Policy fork source", "--owner", "verify")
         assert listed_note(source) == {"purpose": "Policy fork source", "owner": "verify"}
     finally:
-        cleared = live.raft("policy", location, "--clear")
+        cleared = live.raft("policy", location, "--require", "none")
     assert cleared.stdout == f"{location}: required note fields purpose,owner -> none\n"
     assert json.loads(live.raft("config").stdout)["policy"][location]["require"] == []
     live.raft("note", source, "--clear")
@@ -641,6 +805,117 @@ def verify_note_policy(location, create, listed_note, fixtures):
     print(f"{location}: host note policy for new, fork, note and recover passed", flush=True)
 
 
+def verify_retention(location, create, exists, worker):
+    """Set a 1-hour stopped-box retention, prove the worker deletes only a box stopped
+    past it, records stop times Raft missed and logs to the journal, then clear it."""
+    host = settings(location)["ssh"]
+    started = int(time.time())
+    running = create("--memory", "1GiB", "--purpose", "Retention: running box")
+    kept = create("--memory", "1GiB", "--purpose", "Retention: recently stopped box")
+    expired = create("--memory", "1GiB", "--purpose", "Retention: long stopped box")
+    unrecorded = create("--memory", "1GiB", "--purpose", "Retention: box stopped outside Raft")
+    live.raft("stop", kept)
+    live.raft("stop", expired)
+    # A native stop records no stop time; the worker starts that box's clock.
+    incus(location, "stop", parse_handle(unrecorded)[1], "--timeout", "30", locked=True)
+
+    def stopped_at(handle):
+        return incus(
+            location, "config", "get", parse_handle(handle)[1], "user.raft.stopped-at"
+        ).strip()
+
+    def tick(done):
+        # The staged worker runs once; the installed one runs on its 15-second timer.
+        if worker:
+            transaction(location, 'exec python3 "$1"', worker)
+        deadline = time.monotonic() + 50
+        while not done():
+            if time.monotonic() > deadline:
+                raise RuntimeError("Host expiry worker did not apply the retention policy")
+            time.sleep(1)
+
+    def listed():
+        lines = live.raft("list", "--location", location).stdout.splitlines()
+        return {record["box"]: record for record in map(json.loads, lines)}
+
+    if worker:
+        # Older workers ignore retention, so the CLI refuses to set it for them.
+        rejected = live.raft("policy", location, "--delete-stopped-after", "1", check=False)
+        assert rejected.returncode and "deploy the current Raft checkout" in rejected.stderr
+        incus(
+            location,
+            "project",
+            "set",
+            "raft",
+            f"{RETENTION_KEY}=1",
+            f"{RETENTION_SINCE_KEY}={int(time.time())}",
+            locked=True,
+        )
+    else:
+        changed = live.raft("policy", location, "--delete-stopped-after", "1")
+        assert changed.stdout == f"{location}: delete stopped boxes after never -> 1 hour\n"
+    try:
+        policy = json.loads(live.raft("config").stdout)["policy"][location]
+        assert policy["deleteStoppedAfterHours"] == 1, policy
+        since = policy["deleteStoppedAfterSince"]
+        assert started <= since <= time.time(), policy
+        # A box stopped before the policy gets the full hour from the change.
+        records = listed()
+        assert records[kept]["deleteAt"] == str(since + 3600), records[kept]
+        assert records[running]["deleteAt"] == "", records[running]
+        # Hours are the smallest unit, so backdate the policy and one stop by two
+        # hours instead of waiting: only that box is then past the retention.
+        backdated = int(time.time()) - 2 * 3600
+        incus(location, "project", "set", "raft", f"{RETENTION_SINCE_KEY}={backdated}", locked=True)
+        incus(
+            location,
+            "config",
+            "set",
+            parse_handle(expired)[1],
+            f"user.raft.stopped-at={backdated}",
+            locked=True,
+        )
+        tick(lambda: not exists(expired) and stopped_at(unrecorded))
+        assert exists(kept) and exists(running) and exists(unrecorded)
+        # The clock of a box stopped outside Raft starts when the worker sees it.
+        assert int(stopped_at(unrecorded)) >= started
+        assert listed()[unrecorded]["deleteAt"] == str(int(stopped_at(unrecorded)) + 3600)
+        deleted = f"Deleted box {parse_handle(expired)[1]}, stopped since "
+        command = ["sudo", "-n", "journalctl", "-t", "raft", "--since", f"@{started}", "-o", "cat"]
+        deadline = time.monotonic() + 10
+        while True:
+            journal = subprocess.check_output(["ssh", "-T", host, shlex.join(command)], text=True)
+            if deleted in journal or time.monotonic() > deadline:
+                break
+            time.sleep(1)
+        assert deleted in journal, journal
+        if not worker:
+            assert "Stopped-box retention: hours: '' -> '1'" in journal, journal
+    finally:
+        if worker:
+            incus(
+                location,
+                "project",
+                "set",
+                "raft",
+                f"{RETENTION_KEY}=",
+                f"{RETENTION_SINCE_KEY}=",
+                locked=True,
+            )
+        else:
+            cleared = live.raft("policy", location, "--delete-stopped-after", "never")
+    if not worker:
+        assert cleared.stdout == f"{location}: delete stopped boxes after 1 hour -> never\n"
+    policy = json.loads(live.raft("config").stdout)["policy"][location]
+    assert policy["deleteStoppedAfterHours"] is None, policy
+    assert listed()[kept]["deleteAt"] == ""
+    print(
+        f"{location}: stopped-box retention deleted only the expired box, "
+        "recorded missing stop times, logged and cleared",
+        flush=True,
+    )
+
+
 def verify(location, isolated_worker=False):
     if inventory(location):
         raise RuntimeError(
@@ -681,7 +956,7 @@ def verify(location, isolated_worker=False):
             ],
             text=True,
         ).strip()
-        if protocol == "1":
+        if protocol == "2":
             raise RuntimeError(
                 "--isolated-worker targets older deployments; use normal verification on a current host"
             )
@@ -965,6 +1240,7 @@ def verify(location, isolated_worker=False):
             f"{location}: disposable deletion, preview/apply retention and exclusions passed",
             flush=True,
         )
+        verify_retention(location, create, exists, worker)
     finally:
         for handle in reversed(fixtures):
             if exists(handle):
@@ -996,6 +1272,7 @@ if __name__ == "__main__":
     args = parser.parse_args()
     verify_location_notes()
     verify_controller_defaults()
+    verify_retention_worker()
     verify_host_preflight()
     verify_archives()
     if args.location:

package/deploy/verify-limits.py

@@ -108,7 +108,8 @@ def policy_checks():
 
 
 def command(*args):
-    return subprocess.check_output([*CLI, *args], text=True).strip()
+    # The private controller HOME keeps this controller's defaults for new out.
+    return subprocess.check_output([*CLI, *args], text=True, env=live.CONTROLLER_ENV).strip()
 
 
 def report(location):

package/deploy/verify-live.py

@@ -7,6 +7,7 @@ import os
 import pty
 import select
 import shlex
+import shutil
 from pathlib import Path
 import socket
 import subprocess
@@ -19,11 +20,24 @@ ROOT = Path(__file__).resolve().parent.parent
 CLI = [sys.executable, str(ROOT / "raft.py")]
 USER_AGENT = "OpenAI File Downloader, XaiImageApiFetch/1.0"
 sys.path.insert(0, str(ROOT))
-from raft import configuration, incus, remote, settings, transaction  # noqa: E402
+from raft import CONFIG, configuration, incus, remote, settings, transaction  # noqa: E402
+
+# Suites run the CLI in a private HOME that holds only the location inventory, so
+# this controller's defaults for new (config.json) never reach them: a disposable
+# default would delete boxes a suite stops and expects to keep, and size defaults
+# would change asserted limits. OpenSSH reads keys and aliases from the passwd
+# home, not HOME. The inventory is copied once; suites that edit it use their own.
+CONTROLLER_HOME = tempfile.TemporaryDirectory(prefix="raft-verify-home-")
+if CONFIG.exists():
+    (Path(CONTROLLER_HOME.name) / ".config/raft").mkdir(parents=True)
+    shutil.copy2(CONFIG, Path(CONTROLLER_HOME.name) / ".config/raft/incus.json")
+CONTROLLER_ENV = {**os.environ, "HOME": CONTROLLER_HOME.name}
 
 
 def raft(*arguments, check=True, env=None):
-    response = subprocess.run([*CLI, *arguments], capture_output=True, text=True, env=env)
+    response = subprocess.run(
+        [*CLI, *arguments], capture_output=True, text=True, env=env or CONTROLLER_ENV
+    )
     if check and response.returncode:
         raise RuntimeError(f"Raft {arguments!r} failed: {response.stderr.strip()}")
     return response
@@ -352,7 +366,9 @@ def browser_session(box):
                 assert.equal(response.result.errorText, undefined);
                 socket.send(JSON.stringify(viewerProof));
             } else if (response.id === 4) {
-                const viewer = JSON.parse(response.result.result.value);
+                // While noVNC is still loading, the evaluation can return no value.
+                const value = response.result.result.value;
+                const viewer = typeof value === 'string' ? JSON.parse(value) : {connected: false};
                 if (!viewer.connected) {
                     setTimeout(() => socket.send(JSON.stringify(viewerProof)), 200); return;
                 }

package/docs/contract.md

@@ -35,7 +35,7 @@ Raft manages persistent unprivileged native-architecture Incus system containers
 | Host identification | Configuration requires named SSH targets using letters, digits, underscores, or hyphens. |
 | Qualified handles | Every handle includes location and instance name (`<location>:<name>`). |
 | Image pinning | Workspace creation requires the immutable 64-hex image fingerprint, not an updating alias. |
-| Lifecycle defaults | Built-in defaults are the first configured location, 1 CPU, 2 GiB RAM, the shared pool, and 600-second lifetime. `~/.config/raft/config.json` can override them for `new` only; explicit flags win. Boxes from a snapshot (`new --from`) and forks keep the source's CPU, RAM and disk instead of any default, configured or built-in; each flag overrides only its own value. `--cpu` takes any positive whole number, `--memory` any positive whole GiB (`16GiB`), `--disk` a positive whole GiB or `shared`, and TTL 60 to 2,592,000 seconds (30 days). |
+| Lifecycle defaults | Built-in defaults are the first configured location, 1 CPU, 2 GiB RAM, the shared pool, a persistent box and 600-second lifetime. `~/.config/raft/config.json` can override them for `new` only; explicit flags win, and `--persistent` overrides a `disposable` default. Boxes from a snapshot (`new --from`) and forks keep the source's CPU, RAM and disk instead of any default, configured or built-in; each flag overrides only its own value. `--cpu` takes any positive whole number, `--memory` any positive whole GiB (`16GiB`), `--disk` a positive whole GiB or `shared`, and TTL 60 to 2,592,000 seconds (30 days). |
 | Box count and size | Raft limits neither how many boxes a host holds nor their size. CPU or RAM limits larger than the whole host (CPU count, or `MemTotal` rounded up to whole GiB) are refused under the host lock before any change, because Incus would clip them silently. Other failures come from Incus, such as too little pool space. |
 | TTL calculation | Running TTL begins after launch completes, using the host clock. Cold image preparation does not consume TTL. |
 | Host locking | Creation, lifecycle transitions, and expiry are serialized by host file lock (`/run/lock/raft-incus.lock`). Noninteractive transactions close controller stdin so commands cannot wait for piped input while holding this lock. Exec, SSH, and file-transfer paths remain unchanged. Piped input to `raft exec` is unsupported. Native Incus admin calls bypass CLI policy. |
@@ -44,14 +44,15 @@ Raft manages persistent unprivileged native-architecture Incus system containers
 
 `raft locations` lists configured locations as JSONL with `location`, `default` and `note` fields without contacting hosts. `raft note LOCATION TEXT` and `raft note LOCATION --clear` edit the optional `note` field in the controller configuration atomically, preserving location order and mode `0600`. Notes are advisory guidance; Raft shows them in `locations`, `capacity` and on stderr from `new`, and never enforces them.
 
-Boxes accept an optional note with a fixed template: `purpose` (required), `owner` and `next`, all nonempty strings. `raft new --purpose TEXT [--owner TEXT] [--next TEXT]` and `raft note BOX ...` store it as JSON in the box's `user.raft.note` Incus key; `raft note BOX --clear` removes it. Each edit replaces the whole note. `raft list` reports it as `note`, or `null`, and reports each box's `disk` as `"shared"` or its own disk size. `new` always writes the key, so `new --from` never inherits a snapshot's note. Forks and recovered backups keep it unless `fork` or `recover` gets note flags, which replace the whole note. Box notes never change lifecycle behavior.
+Boxes accept an optional note with a fixed template: `purpose` (required), `owner` and `next`, all nonempty strings. `raft new --purpose TEXT [--owner TEXT] [--next TEXT]` and `raft note BOX ...` store it as JSON in the box's `user.raft.note` Incus key; `raft note BOX --clear` removes it. Each edit replaces the whole note. `raft list` reports it as `note`, or `null`, reports each box's `disk` as `"shared"` or its own disk size, and reports `deleteAt`, the Unix time retention deletes a stopped box, or `""`. `new` always writes the key, so `new --from` never inherits a snapshot's note. Forks and recovered backups keep it unless `fork` or `recover` gets note flags, which replace the whole note. Box notes never change lifecycle behavior.
 
-## Controller defaults and note policies
+## Controller defaults and host policies
 
-- **Controller defaults**: `~/.config/raft/config.json` holds `{"defaults": {...}}` with optional `location`, `ttl`, `cpu`, `memory` and `disk` for `new`. Precedence is explicit flag, then configured default, then built-in default. `cpu`, `memory` and `disk` apply only to boxes from the image: `new --from` keeps the snapshot's location, CPU, RAM and disk unless a flag overrides them, and `--disk shared` is refused there. A `disk` default on a host without `raft-disks`, or larger than its unreserved space, fails like `--disk` and names `defaults.disk` as the source. Other commands take no defaults. A value is valid exactly when the same text is valid for the flag. Unknown keys, bad values and malformed JSON fail `new` with the file path and key, before any host access. `raft config set/unset` validate first, write atomically under a controller lock, keep mode `0600`, print the change and can repair the key they edit. The file is separate from `incus.json`, so location names never collide with setting names.
-- **Note policy storage**: `raft policy LOCATION --require FIELDS` stores required box note fields, which must include `purpose`, in the host's Incus project key `user.raft.require`; `--clear` empties it. Each change runs under the host lock, prints the old and new fields, records UTC time, the SSH account (`SUDO_USER`) and the self-reported controller user and hostname in `user.raft.require.changed`, and logs to the host journal with tag `raft`. Redeployment leaves these keys unchanged.
+- **Controller defaults**: `~/.config/raft/config.json` holds `{"defaults": {...}}` with optional `location`, `ttl`, `cpu`, `memory`, `disk` and `disposable` for `new`. Precedence is explicit flag, then configured default, then built-in default. `cpu`, `memory` and `disk` apply only to boxes from the image: `new --from` keeps the snapshot's location, CPU, RAM and disk unless a flag overrides them, and `--disk shared` is refused there. `ttl` and `disposable` apply to `new --from` as well, because a snapshot carries no lifecycle; `new --disposable` and `--persistent` are mutually exclusive overrides. `fork` takes no defaults and is persistent without `--disposable`. A `disk` default on a host without `raft-disks`, or larger than its unreserved space, fails like `--disk` and names `defaults.disk` as the source. Other commands take no defaults. A value is valid exactly when the same text is valid for the flag. Unknown keys, bad values and malformed JSON fail `new` with the file path and key, before any host access. `raft config set/unset` validate first, write atomically under a controller lock, keep mode `0600`, print the change and can repair the key they edit. The file is separate from `incus.json`, so location names never collide with setting names.
+- **Policy storage**: `raft policy LOCATION [--require FIELDS|none] [--delete-stopped-after HOURS|never]` sets either host policy or both; each option changes only its own. Required box note fields, which must include `purpose`, live in the Incus project key `user.raft.require`, and `--require none` empties it. Stopped-box retention lives in `user.raft.delete-stopped-after` as a positive whole number of hours, with the time it was set in `user.raft.delete-stopped-after.since`; `never` empties both. Both values are validated before any host change. Each change runs under the host lock, prints the old and new value, records UTC time, the SSH account (`SUDO_USER`) and the self-reported controller user and hostname in the policy's `.changed` key, and logs to the host journal with tag `raft`. Redeployment leaves these keys unchanged.
 - **Note policy enforcement**: `new` (including `--from`), `fork`, `recover` and `raft note BOX` check the note the box will carry. The controller reads the policy first and fails with the missing flags and a complete corrected command before changing anything. The host repeats the check inside the transaction that writes the note, under the host lock, so a concurrent policy change cannot be missed. That transaction also holds the oversize and own-disk reservation checks; all of them run before Incus creates anything. `fork` checks the given note or, without one, the source note read under the lock. `recover` checks a given note before upload; an archived note is checked after import, and a failing import is deleted in the same transaction. `note --clear` and replacement notes that drop a required field are rejected.
-- **Policy scope**: The policy is a guardrail for agents, not an access control. Anyone with host SSH and sudo can change it or create boxes with native Incus commands, and controllers without policy support do not check it. `raft config` reports every location's policy and last change.
+- **Retention enforcement**: The host expiry worker deletes a stopped Raft box, with its snapshots, once the hours have passed since the later of its `user.raft.stopped-at` and the policy's `.since` time, so setting, shortening or renewing a policy never deletes a box sooner than its hours after the change. It runs on the 15-second timer and on `raft gc`, under the host lock, and writes each deletion to the host journal with tag `raft`. Running boxes are never touched. A stopped Raft box without `user.raft.stopped-at`, from a guest poweroff, a native stop, a failed start or `raft recover`, gets the worker's current time as its stop time, never an earlier guess, whether or not a policy is set. A malformed retention value makes the worker delete nothing by retention, still enforce deadlines, and exit nonzero; the CLI reports it with the fix. Setting retention requires the current worker protocol; clearing it does not.
+- **Policy scope**: Policies are guardrails for agents, not access controls. Anyone with host SSH and sudo can change them or create boxes with native Incus commands, and controllers without note policy support do not check it. A native `incus start` leaves a stale stop time that retention trusts. `raft config` reports every location's policies and last changes.
 
 `raft capacity [--location NAME] [--json]` inspects hosts without modifying state. Text and JSON (`notice`) lead with: "Advisory estimate only. Raft enforces no box count or size budget; create the boxes your task needs." It reports effective CPUs, total and available RAM, saved and active box counts, active resource limits, headroom, free space in the host disk and the shared pool, and for `raft-disks` its data space, reserved and used bytes and metadata use (`boxDiskPool`, `null` when own box disks are not enabled).
 
@@ -64,12 +65,12 @@ Boxes accept an optional note with a fixed template: `purpose` (required), `owne
 
 - **Stop and resume**: Stop terminates guest processes and preserves disk contents; stopped boxes retain disk but do not reserve guest RAM. Resume starts the container anew with an explicit TTL without restoring process memory.
 - **Lifetime extension**: `raft extend --ttl` resets a running box's deadline from current host time without restarting; stopped boxes require `raft resume`.
-- **Host worker contract**: `doctor`, disposable creation/fork, and retention cleanup require the current worker protocol. `doctor` checks it without running expiry or changing boxes, and reports the failing location. A mismatch fails before executing deletion policy or creating a disposable box; deploy the current checkout explicitly. Installing a new CLI does not update host workers.
-- **Disposable lifecycle**: `new --disposable` opts a box into deletion on stop or TTL expiry, including its snapshots. Clones and recovered boxes default to persistent; fork accepts an explicit `--disposable`.
-- **Retention cleanup**: `prune --older-than DAYS` previews persistent stopped boxes with a recorded stop timestamp. `--yes` deletes matching boxes under the host lifecycle lock. Running boxes and boxes without a recorded stop time are excluded. No automatic persistent-box deletion is enabled.
+- **Host worker contract**: `doctor`, disposable creation/fork, prune and setting a retention policy require the current worker protocol (2 since 0.6.0, which added retention). `doctor` checks it without running expiry or changing boxes, and reports the failing location. A mismatch fails before executing deletion policy or creating a disposable box; deploy the current checkout explicitly. Installing a new CLI does not update host workers.
+- **Disposable lifecycle**: `new --disposable`, or a configured `disposable` default without `--persistent`, opts a box into deletion on stop or TTL expiry, including its snapshots. Clones and recovered boxes default to persistent whatever the default; fork accepts an explicit `--disposable`.
+- **Manual cleanup**: `prune --older-than DAYS` previews persistent stopped boxes with a recorded stop timestamp. `--yes` deletes matching boxes under the host lifecycle lock. Running boxes and boxes without a recorded stop time are excluded. Automatic deletion of persistent stopped boxes happens only under a host retention policy.
 - **Resizing**: Resume and fork accept optional CPU/RAM overrides, preserving omitted values. Resume validates stopped state under the lock before changing sizing. Neither changes a box's disk.
-- **Expiration cleanup**: The host timer checks expirations every 15 seconds, stopping expired persistent containers and deleting expired disposable containers. Expiry may be delayed by graceful shutdown, long locked transactions, and host outage until startup.
-- **Destroy**: Removes the container instance and its snapshots, releasing storage. Destroy confirms absence; SSH failure is not proof of deletion. Disposable boxes created with `new --disposable` are deleted on explicit stop or expiry. Persistent boxes still require explicit destroy for deletion.
+- **Expiration cleanup**: The host timer checks expirations every 15 seconds, stopping expired persistent containers, deleting expired disposable containers and applying stopped-box retention. Expiry may be delayed by graceful shutdown, long locked transactions, and host outage until startup.
+- **Destroy**: Removes the container instance and its snapshots, releasing storage. Destroy confirms absence; SSH failure is not proof of deletion. Disposable boxes created with `new --disposable` are deleted on explicit stop or expiry. Persistent boxes still require explicit destroy, prune or a retention policy for deletion.
 - **Execution transport**: Incus exec and file APIs travel over authenticated host SSH without a public Incus listener or injected credentials. Interactive SSH runs a PTY over host SSH and Incus exec.
 - **State inspection and tunnels**: `raft list` and `raft info` read native metadata without guest process counters during shutdown; tunnels use the managed NIC.
 - **Resource reporting**: `raft usage` reports running cgroup memory, CPU microseconds, CPU affinity, and filesystem space with `filesystemScope`: the shared pool, or `own SIZE box disk`. It provides no billing.

package/docs/first-box.mdx

@@ -83,4 +83,4 @@ raft destroy "$box"
 - [Troubleshooting](./guides/troubleshooting.mdx): Diagnose common errors and inspect host services.
 - [Upgrades and image changes](./guides/upgrades.mdx): Update the controller, hosts, and image pins.
 
-For temporary work, create with `--disposable`. Stopping or expiry then deletes the box and its snapshots. For persistent boxes, `raft prune --older-than 7` previews old stopped workspaces; add `--yes` only after reviewing the candidates.
+For temporary work, create with `--disposable`. Stopping or expiry then deletes the box and its snapshots. For persistent boxes, `raft prune --older-than 7` previews old stopped workspaces; add `--yes` only after reviewing the candidates. A host can also delete boxes left stopped automatically; `raft config` shows each location's `deleteStoppedAfterHours`.

package/docs/guides/backup-recovery.mdx

@@ -53,7 +53,7 @@ echo "Recovered workspace: $recovered"
 
 - **Architecture compatibility**: Target host must match the native CPU architecture (ARM64 to ARM64, AMD64 to AMD64). Cross-architecture recovery is not supported.
 - **Fresh network identity**: The container receives a new name in project `raft`, a new MAC address on `rfbr0`, and no inherited TTL deadline.
-- **Stopped initial state**: Recovered workspaces remain stopped until started with `raft resume "$recovered" --ttl <SECONDS>`.
+- **Stopped initial state**: Recovered workspaces remain stopped until started with `raft resume "$recovered" --ttl <SECONDS>`. Under a host [retention policy](./config-policy.mdx#delete-boxes-left-stopped), the recovered box's clock starts when the host worker first sees it, within about 15 seconds; resume it before the policy's hours pass.
 - **Disk placement**: A backup of a box with its own disk returns to an own disk of the same size in `raft-disks`. The destination needs own box disks enabled and enough unreserved pool space for the box and each of its snapshots; otherwise recovery fails before importing. Other backups land in the shared pool.
 - **Interrupted recovery**: Follow [Host crash during recovery](./troubleshooting.mdx#host-crash-during-recovery). Inspect the reported handle and container state before deleting any staging file.