@microck/raft 0.5.0 → 0.7.0

package/raft.py

@@ -24,8 +24,11 @@ import urllib.request
 import uuid
 
 import raft_files
+import raft_run
+import raft_jobs
+import raft_output
 
-__version__ = "0.5.0"
+__version__ = "0.7.0"
 
 CONFIG = Path.home() / ".config/raft/incus.json"
 # Controller defaults for new live apart from the location inventory, so location
@@ -195,7 +198,14 @@ def box_disk(box):
 # guardrail for agents: SSH and sudo on the host bypass it, and controllers
 # older than this check never run it.
 POLICY_KEY = "user.raft.require"
-POLICY_CHANGED_KEY = "user.raft.require.changed"
+# Stopped-box retention, the second host policy: the host expiry worker deletes
+# raft boxes that have been stopped for this many hours. Its .since key holds
+# when the policy was last set, in epoch seconds: the retention clock starts at
+# the later of a box's stop and that time, so setting or shortening a policy
+# never deletes a box sooner than the given hours after the change. Mirrors
+# RETENTION_KEY in deploy/raft-expire.py, which enforces it.
+RETENTION_KEY = "user.raft.delete-stopped-after"
+RETENTION_SINCE_KEY = RETENTION_KEY + ".since"
 
 # Host half of the check, run with the required fields and the note the box
 # will carry. Kept short; the controller builds the detailed message.
@@ -218,20 +228,53 @@ READ_NOTE_POLICY = f"require=$(incus project get raft {POLICY_KEY})\n"
 CHECK_NOTE = "python3 -c " + shlex.quote(CHECK_NOTE_CODE) + ' "$require" "$note"'
 
 
-def note_policy(location):
-    """Return a location's required note fields and its last recorded policy change."""
+def host_policy(location):
+    """Return a location's host policies and their last recorded changes, as raft config shows them."""
     # `incus query` rejects the --project flag that remote() always passes.
     projects = json.loads(incus(location, "project", "list", "--format", "json"))
-    project = next(item for item in projects if item["name"] == PROJECT)
-    stored = project["config"].get(POLICY_KEY, "")
+    config = next(item for item in projects if item["name"] == PROJECT)["config"]
+    return parse_host_policy(location, config)
+
+
+def parse_host_policy(location, config):
+    """Validate native policy values before displaying or enforcing them."""
+    stored = config.get(POLICY_KEY, "")
     required = [field for field in stored.split(",") if field]
     unknown = [field for field in required if field not in BOX_NOTE_FIELDS]
     if unknown:
         raise ValueError(
             f"{location}: host policy {POLICY_KEY}={stored!r} names unknown note fields; "
-            f"fix it with raft policy {location} --require FIELDS or --clear"
+            f"fix it with raft policy {location} --require FIELDS or --require none"
         )
-    return {"require": required, "changed": project["config"].get(POLICY_CHANGED_KEY, "")}
+    hours = config.get(RETENTION_KEY, "")
+    since = config.get(RETENTION_SINCE_KEY, "")
+    # Raft writes both as whole numbers; anything else came from a native Incus edit.
+    if hours and not (re.fullmatch(r"[1-9][0-9]*", hours) and re.fullmatch(r"[0-9]+", since)):
+        raise ValueError(
+            f"{location}: host policy {RETENTION_KEY}={hours!r} ({RETENTION_SINCE_KEY}={since!r}) "
+            f"is not a whole number of hours; fix it with raft policy {location} "
+            "--delete-stopped-after HOURS or --delete-stopped-after never"
+        )
+    return {
+        "require": required,
+        "requireChanged": config.get(POLICY_KEY + ".changed", ""),
+        "deleteStoppedAfterHours": int(hours) if hours else None,
+        "deleteStoppedAfterSince": int(since) if hours else None,
+        "deleteStoppedAfterChanged": config.get(RETENTION_KEY + ".changed", ""),
+    }
+
+
+def delete_at(box, policy):
+    """When the host worker deletes a stopped box under the retention policy, or None.
+
+    Must match the deletion rule in deploy/raft-expire.py. A stopped box without
+    a recorded stop time has no deadline until the worker records one.
+    """
+    hours = policy["deleteStoppedAfterHours"]
+    stopped = box["config"].get("user.raft.stopped-at")
+    if box["status"] != "Stopped" or hours is None or not stopped:
+        return None
+    return max(int(stopped), policy["deleteStoppedAfterSince"]) + hours * 3600
 
 
 def note_problem(location, required, note, command, hint=""):
@@ -252,13 +295,15 @@ def note_problem(location, required, note, command, hint=""):
 
 def require_note(location, note, command, hint=""):
     """Fail before any host change when note lacks a field the location requires."""
-    problem = note_problem(location, note_policy(location)["require"], note, command, hint)
+    problem = note_problem(location, host_policy(location)["require"], note, command, hint)
     if problem:
         raise ValueError(problem)
 
 
 def note_fields(text):
-    """Parse --require FIELDS into BOX_NOTE_FIELDS order."""
+    """Parse --require FIELDS into BOX_NOTE_FIELDS order; none means no required fields."""
+    if text == "none":
+        return []
     fields = [field.strip() for field in text.split(",") if field.strip()]
     unknown = [field for field in fields if field not in BOX_NOTE_FIELDS]
     if unknown or not fields:
@@ -271,23 +316,69 @@ def note_fields(text):
     return [field for field in BOX_NOTE_FIELDS if field in fields]
 
 
-def set_policy(location, fields):
-    """Set (fields) or clear (None) a location's note policy and record the change on the host."""
-    settings(location)
-    value = ",".join(fields or [])
+def retention_hours(text):
+    """Parse --delete-stopped-after HOURS: a positive whole number, or never for no retention."""
+    if text == "never":
+        return None
+    if not re.fullmatch(r"[1-9][0-9]*", text):
+        raise ValueError(
+            "Give a positive whole number of hours, e.g. --delete-stopped-after 48, or never"
+        )
+    return int(text)
+
+
+def set_policy(location, key, value, journal, check=""):
+    """Set one host policy key to value, empty to clear it, and record the change.
+
+    Returns the previous value. journal names the policy in the host journal entry;
+    check is shell that must pass first, under the same lock.
+    """
     # Who and when is recorded on the host beside the policy and in its journal.
     # The controller identity is self-reported; SUDO_USER is the SSH account.
-    before = transaction(
+    # Retention also records when it was set, for the worker's clock (see RETENTION_KEY).
+    since = f' {RETENTION_SINCE_KEY}="$([ -z "$1" ] || date +%s)"' if key == RETENTION_KEY else ""
+    return transaction(
         location,
-        f"before=$(incus project get raft {POLICY_KEY})\n"
+        check + f"before=$(incus project get raft {key})\n"
         'changed="$(date -u +%Y-%m-%dT%H:%M:%SZ) by ${SUDO_USER:-root} from $2"\n'
-        f'incus project set raft {POLICY_KEY}="$1" {POLICY_CHANGED_KEY}="$changed"\n'
-        "logger -t raft \"Box note policy: required fields '$before' -> '$1' ($changed)\"\n"
+        f'incus project set raft {key}="$1" {key}.changed="$changed"{since}\n'
+        f"logger -t raft \"{journal}: '$before' -> '$1' ($changed)\"\n"
         "printf '%s\\n' \"$before\"",
         value,
         f"{getpass.getuser()}@{socket.gethostname()}",
     ).strip()
-    print(f"{location}: required note fields {before or 'none'} -> {value or 'none'}")
+
+
+def change_policy(args):
+    """Apply raft policy: each given option sets or clears its own policy independently."""
+    if args.require is None and args.delete_stopped_after is None:
+        raise ValueError("Give --require FIELDS|none, --delete-stopped-after HOURS|never, or both")
+    settings(args.location)
+    # Parse every option before the first host change. Retention goes first: its
+    # worker check is the one step that can still refuse.
+    fields = None if args.require is None else note_fields(args.require)
+    hours = (
+        None if args.delete_stopped_after is None else retention_hours(args.delete_stopped_after)
+    )
+    if args.delete_stopped_after is not None:
+        value = "" if hours is None else str(hours)
+        before = set_policy(
+            args.location,
+            RETENTION_KEY,
+            value,
+            "Stopped-box retention: hours",
+            # Only a current worker enforces retention; clearing is safe on any worker.
+            check=CHECK_WORKER if hours else "",
+        )
+
+        def shown(stored):
+            return "never" if not stored else stored + (" hour" if stored == "1" else " hours")
+
+        print(f"{args.location}: delete stopped boxes after {shown(before)} -> {shown(value)}")
+    if fields is not None:
+        value = ",".join(fields)
+        before = set_policy(args.location, POLICY_KEY, value, "Box note policy: required fields")
+        print(f"{args.location}: required note fields {before or 'none'} -> {value or 'none'}")
 
 
 def receive(command, destination, *, stdin=None):
@@ -298,11 +389,11 @@ def receive(command, destination, *, stdin=None):
         ) as process:
             try:
                 shutil.copyfileobj(process.stdout, destination)
+                returncode = process.wait()
             except BaseException:
                 process.kill()
                 process.wait()
                 raise
-            returncode = process.wait()
         errors.seek(0)
         return subprocess.CompletedProcess(command, returncode, None, errors.read())
 
@@ -571,17 +662,44 @@ def inventory(location):
     ]
 
 
+# These probes retain --fast so stopped/stopping guests never need live counters.
+HOST_BOXES = (
+    "project = "
+    + repr(PROJECT)
+    + "\n"
+    + """
+print(json.dumps(json.loads(subprocess.run(['incus', '--project', project, 'list', '--fast', '--format', 'json'],
+                     check=True, capture_output=True).stdout)))
+"""
+)
+HOST_INVENTORY = (
+    "project = "
+    + repr(PROJECT)
+    + "\n"
+    + """import json, subprocess
+projects = json.loads(subprocess.run(['incus', '--project', project, 'project', 'list', '--format', 'json'],
+                                    check=True, capture_output=True).stdout)
+boxes = json.loads(subprocess.run(['incus', '--project', project, 'list', '--fast', '--format', 'json'],
+                                 check=True, capture_output=True).stdout)
+print(json.dumps({'policy': next(p for p in projects if p['name'] == project)['config'], 'boxes': boxes}))
+"""
+)
+
+
 REQUIRE_STOPPED = """state=$(incus --project raft list "$1" --fast --format json | python3 -c 'import json,sys; print(next(x["status"] for x in json.load(sys.stdin) if x["name"] == sys.argv[1] and x["config"].get("user.raft") == "true"))' "$1")
 test "$state" = Stopped || { echo 'Stop the source box before resume, snapshot, restore, fork or backup' >&2; exit 1; }
 """
 
 
+# Refuses a host whose expiry worker speaks another PROTOCOL than this CLI. Bump
+# PROTOCOL here and in deploy/raft-expire.py together whenever the CLI starts to
+# rely on new worker behavior; protocol 2 added stopped-box retention.
 CHECK_WORKER = (
     "python3 -c "
     + shlex.quote(
         "import runpy; "
         "worker = runpy.run_path('/usr/local/lib/raft/expire-worker'); "
-        "\nif worker.get('PROTOCOL') != 1:\n"
+        "\nif worker.get('PROTOCOL') != 2:\n"
         " raise SystemExit('Host expiry worker differs from this CLI; deploy the current Raft checkout first')"
     )
     + "\n"
@@ -831,26 +949,44 @@ def capacity(host, boxes):
     }
 
 
-def host_capacity(location):
+def host_read(location, script):
+    """Run a read-only host probe without taking the lifecycle/expiry lock."""
     response = subprocess.run(
         [
             "ssh",
+            "-n",
             "-T",
             "-o",
             "BatchMode=yes",
             "-o",
             "ConnectTimeout=10",
             settings(location)["ssh"],
-            shlex.join(["sudo", "-n", "python3", "-c", HOST_RESOURCES]),
+            shlex.join(["sudo", "-n", "python3", "-c", script]),
         ],
         capture_output=True,
         text=True,
-        check=True,
     )
+    if response.returncode:
+        raise RuntimeError(f"{location}: {response.stderr.strip()}")
+    return response.stdout
+
+
+def host_inventory(location):
+    # Keep both native reads on the host: network latency costs more than JSON
+    # decoding. Do not lock read-only inventory behind a long backup or creation.
+    report = json.loads(host_read(location, HOST_INVENTORY))
+    policy = parse_host_policy(location, report["policy"])
+    boxes = [box for box in report["boxes"] if box["config"].get("user.raft") == "true"]
+    return policy, boxes
+
+
+def host_capacity(location):
+    resources, boxes = host_read(location, HOST_RESOURCES + HOST_BOXES).splitlines()
+    boxes = [box for box in json.loads(boxes) if box["config"].get("user.raft") == "true"]
     return {
         "location": location,
         "note": settings(location).get("note", ""),
-        **capacity(json.loads(response.stdout), inventory(location)),
+        **capacity(json.loads(resources), boxes),
     }
 
 
@@ -900,34 +1036,34 @@ def print_capacity(report):
 
 
 def doctor(location):
-    print(location + ": " + incus(location, "version").strip())
-    try:
-        transaction(location, CHECK_WORKER)
-    except RuntimeError as error:
-        raise RuntimeError(f"{location}: {error}") from error
-    response = subprocess.run(
-        [
-            "ssh",
-            "-T",
-            "-o",
-            "BatchMode=yes",
-            "-o",
-            "ConnectTimeout=10",
-            settings(location)["ssh"],
-            "systemctl is-active incus raft-expire.timer raft-network.service && df -h /",
-        ],
-        capture_output=True,
-        text=True,
-        check=True,
+    # Health checks are sequential on the host, not separate SSH handshakes.
+    # Only the worker compatibility check takes the lifecycle lock, as before.
+    script = "import json, subprocess, sys\nworker_check = " + repr(CHECK_WORKER) + "\n"
+    script += (
+        f"project = {PROJECT!r}\nshared_pool = {SHARED_POOL!r}\nbox_disk_pool = {BOX_DISK_POOL!r}\n"
     )
-    print(response.stdout.strip())
-    print(incus(location, "storage", "info", SHARED_POOL).strip())
-    # Own box disks are opt-in, so a host without their pool is healthy.
-    if BOX_DISK_POOL in incus(location, "storage", "list", "--format", "csv", "-c", "n").split():
-        print(incus(location, "storage", "info", BOX_DISK_POOL).strip())
-    else:
-        print("own box disks: not enabled (redeploy with --disk-pool SIZE)")
-    print(f"{len(inventory(location))} saved boxes; system containers (shared host kernel)")
+    script += """
+def checked(*command):
+    response = subprocess.run(command, capture_output=True, text=True)
+    if response.returncode:
+        sys.exit(response.stderr.strip() or response.stdout.strip())
+    return response.stdout.strip()
+print(checked('incus', '--project', project, 'version'))
+checked('flock', '/run/lock/raft-incus.lock', 'sh', '-ec', worker_check)
+print(checked('systemctl', 'is-active', 'incus', 'raft-expire.timer', 'raft-network.service'))
+print(checked('df', '-h', '/'))
+print(checked('incus', '--project', project, 'storage', 'info', shared_pool))
+pools = checked('incus', '--project', project, 'storage', 'list', '--format', 'csv', '-c', 'n').split()
+if box_disk_pool in pools:
+    print(checked('incus', '--project', project, 'storage', 'info', box_disk_pool))
+else:
+    print('own box disks: not enabled (redeploy with --disk-pool SIZE)')
+boxes = json.loads(checked('incus', '--project', project, 'list', '--fast', '--format', 'json'))
+print(str(sum(box['config'].get('user.raft') == 'true' for box in boxes))
+      + ' saved boxes; system containers (shared host kernel)')
+"""
+    print(location + ": ", end="", flush=True)
+    print(host_read(location, script), end="")
 
 
 def transaction(location, script, *arguments, stdout=None):
@@ -962,6 +1098,57 @@ def transaction(location, script, *arguments, stdout=None):
     return response.stdout.decode() if response.stdout is not None else ""
 
 
+def sync_source(args, location, name):
+    """Use rsync's checksum/delta protocol inside the guest, never on the host tree."""
+    if shutil.which("rsync") is None:
+        raise ValueError("sync requires rsync on the controller; install rsync first")
+    source = Path(args.source).resolve(strict=True)
+    if not source.is_dir():
+        raise ValueError("Sync source must be a directory")
+    destination = raft_run.guest_path(args.destination)
+    if destination == "/":
+        raise ValueError("Sync destination must not be the guest root directory")
+    selected = None if args.all else raft_run.git_files(source, all_option="--all")
+    filters = None
+    if selected is not None:
+        # Recursive traversal never follows directory symlinks. A files-from list
+        # would follow symlink ancestors when a tracked directory was replaced.
+        allowed = set(selected)
+        for path in selected:
+            allowed.update(str(parent) for parent in Path(path).parents if str(parent) != ".")
+        rules = []
+        for path in sorted(allowed):
+            if any(char in path for char in "*?["):
+                path = re.sub(r"([\\*?\[])", r"\\\1", path)
+            rules.append(b"+ /" + os.fsencode(path))
+        filters = b"\0".join([*rules, b"- *", b""])
+    receiver = shlex.join(
+        ["sudo", "-n", "incus", "--project", PROJECT, "exec", name, "--", "rsync"]
+    )
+    command = [
+        "rsync",
+        "--recursive",
+        "--links",
+        "--safe-links",
+        "--times",
+        "--perms",
+        "--checksum",
+        "--omit-dir-times",
+        "--itemize-changes",
+        "--secluded-args",
+        "--rsync-path",
+        receiver,
+        "-e",
+        "ssh -T -o BatchMode=yes -o ConnectTimeout=10",
+    ]
+    if filters is not None:
+        command += ["--from0", "--filter=merge -"]
+    if args.dry_run:
+        command.append("--dry-run")
+    command += ["--", str(source) + "/", settings(location)["ssh"] + ":" + destination + "/"]
+    return subprocess.run(command, input=filters).returncode
+
+
 def snapshot_name(value):
     if not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_-]{0,63}", value):
         raise ValueError("Invalid snapshot name")
@@ -1018,7 +1205,7 @@ def own_disk(size):
     ]
 
 
-def new(args):
+def new(args, *, run_id=None, on_allocate=None):
     box_fields = box_note(args)
     # Sizes are filled here only for boxes from the image; a box from a snapshot
     # keeps the snapshot's CPU, RAM and disk unless a flag overrides them.
@@ -1047,10 +1234,13 @@ def new(args):
         print(f"{args.location} note: {note}", file=sys.stderr)
     require_note(args.location, box_fields, "new")
     name = "rf-" + uuid.uuid4().hex[:16]
+    if on_allocate is not None:
+        on_allocate(args.location, name)
     # Initialize before starting, so a failed start still has its intended policy.
     transaction(
         args.location,
-        (CHECK_WORKER if args.disposable else "")
+        (raft_run.creation_guard(run_id) if run_id else "")
+        + (CHECK_WORKER if args.disposable else "")
         + fits_host(args)
         # Plain boxes from the image stay off raft-disks and skip the check.
         + (
@@ -1069,10 +1259,15 @@ def new(args):
         + CHECK_NOTE
         # Always set the note, so a box created from a snapshot never inherits the
         # source note. The host checks the same value it writes.
-        + '\n"$@" -c user.raft.note="$note"; '
-        'incus --project raft start "$name"; '
-        'incus --project raft config set "$name" '
-        'user.raft.expires="$(( $(date +%s) + lifetime ))"',
+        + '\n"$@" -c user.raft.note="$note"'
+        + (' -c user.raft.expires="$(( $(date +%s) + lifetime ))"' if run_id else "")
+        + '; incus --project raft start "$name"; '
+        + (
+            ""
+            if run_id
+            else 'incus --project raft config set "$name" '
+            'user.raft.expires="$(( $(date +%s) + lifetime ))"'
+        ),
         str(args.ttl),
         name,
         json.dumps(box_fields) if box_fields else "",
@@ -1084,8 +1279,8 @@ def new(args):
         name,
         "-c",
         "user.raft=true",
-        "-c",
-        "user.raft.expires=",
+        *([] if run_id else ["-c", "user.raft.expires="]),
+        *(["-c", f"user.raft.run={run_id}"] if run_id else []),
         "-c",
         "user.raft.stopped-at=",
         "-c",
@@ -1101,15 +1296,15 @@ def new(args):
     return f"{args.location}:{name}"
 
 
-def execute(location, name, command, *, capture=False):
-    return remote(location, ["exec", name, "--cwd", "/workspace", "--", *command], capture=capture)
+def execute(location, name, command, *, capture=False, cwd="/workspace"):
+    return remote(location, ["exec", name, "--cwd", cwd, "--", *command], capture=capture)
 
 
 def transfer_tree(args, location, name):
     worker = Path(raft_files.__file__).read_text()
     with tempfile.TemporaryFile() as archive:
         if args.action == "upload":
-            raft_files.pack_directory(args.source, archive)
+            raft_files.pack_directory(args.source, archive, include=getattr(args, "include", None))
             size = archive.tell()
             archive.seek(0)
             response = remote(
@@ -1308,6 +1503,18 @@ def location_name(value):
     return value
 
 
+def boolean(value):
+    """Parse a true or false setting, e.g. raft config set disposable true."""
+    if value not in {"true", "false"}:
+        raise argparse.ArgumentTypeError("Use true or false")
+    return value == "true"
+
+
+def shown_default(value):
+    """Print a default as it is typed: true and false, not Python's True and False."""
+    return json.dumps(value) if isinstance(value, bool) else value
+
+
 # Options of new that ~/.config/raft/config.json can default, with their built-in
 # defaults and value checks. Values go through str() first, so a configured value
 # is accepted exactly when the same text passed as the flag would be. Adding a
@@ -1319,6 +1526,9 @@ NEW_DEFAULTS = {
     "cpu": (DEFAULT_CPU, cpu_count),
     "memory": (DEFAULT_MEMORY, gib_size),
     "disk": ("shared", disk_size),
+    # new --disposable and --persistent override it. Unlike the sizes it applies
+    # to new --from too: a snapshot carries no lifecycle, new always sets one.
+    "disposable": (False, boolean),
 }
 # A box from a snapshot keeps the snapshot's size, so these defaults, configured
 # or built-in, apply only to boxes created from the image.
@@ -1333,7 +1543,8 @@ def default_value(key, value):
             "supported keys: " + ", ".join(NEW_DEFAULTS)
         )
     try:
-        return NEW_DEFAULTS[key][1](str(value))
+        # JSON true and false read back as the true and false text the CLI accepts.
+        return NEW_DEFAULTS[key][1](str(shown_default(value)))
     except (ValueError, argparse.ArgumentTypeError) as error:
         raise ValueError(
             f"{DEFAULTS_CONFIG}: defaults.{key}={value!r} is invalid: {error}"
@@ -1445,8 +1656,10 @@ def set_default(key, value):
             document.pop("defaults")
 
     edit_json(DEFAULTS_CONFIG, lambda: {"defaults": configured_defaults(key)}, change)
-    before = previous.get(key, "built-in default")
-    print(f"defaults.{key}: {before} -> {'built-in default' if value is None else value}")
+    before = shown_default(previous.get(key, "built-in default"))
+    print(
+        f"defaults.{key}: {before} -> {'built-in default' if value is None else shown_default(value)}"
+    )
 
 
 # GitHub releases are the one source of version truth. The release workflow
@@ -1689,27 +1902,26 @@ def note_options(command):
     )
 
 
-def parser():
-    cli = argparse.ArgumentParser(description=__doc__)
-    cli.add_argument("--version", action="version", version=__version__)
-    commands = cli.add_subparsers(dest="action", required=True)
-    create = commands.add_parser("new")
-    create.add_argument("--location", default=None)
+def creation_options(command, *, lifecycle=True):
+    command.add_argument("--location", default=None)
     # Omitted options take ~/.config/raft/config.json defaults, then NEW_DEFAULTS.
     # With --from, every omitted size comes from the snapshot instead of a default.
-    create.add_argument("--ttl", type=ttl)
-    create.add_argument(
+    if lifecycle:
+        command.add_argument(
+            "--ttl", type=ttl, help="Lifetime in seconds, 60 to 2592000 (default: config, else 600)"
+        )
+    command.add_argument(
         "--cpu",
         type=cpu_count,
         help=f"CPU limit (default: config, else {DEFAULT_CPU}; with --from, the snapshot's)",
     )
-    create.add_argument(
+    command.add_argument(
         "--memory",
         type=gib_size,
         help="RAM limit, e.g. 16GiB "
         f"(default: config, else {DEFAULT_MEMORY}; with --from, the snapshot's)",
     )
-    create.add_argument(
+    command.add_argument(
         "--disk",
         type=disk_size,
         metavar="SIZE",
@@ -1717,13 +1929,62 @@ def parser():
         "or shared for the shared pool "
         "(default: config, else shared; with --from, the snapshot's disk)",
     )
-    create.add_argument("--disposable", action="store_true", help="Delete on stop or expiry")
-    create.add_argument("--from", dest="template", metavar="LOCATION:BOX/SNAPSHOT")
-    note_options(create)
+    if lifecycle:
+        policy = command.add_mutually_exclusive_group()
+        policy.add_argument(
+            "--disposable",
+            dest="disposable",
+            action="store_const",
+            const=True,
+            help="Delete on stop or expiry (default: config, else persistent)",
+        )
+        policy.add_argument(
+            "--persistent",
+            dest="disposable",
+            action="store_const",
+            const=False,
+            help="Keep the disk on stop or expiry, despite a disposable default",
+        )
+    command.add_argument(
+        "--from",
+        dest="template",
+        metavar="LOCATION:BOX/SNAPSHOT",
+        help="Copy a named snapshot on its source host, including filesystem secrets",
+    )
+    note_options(command)
+
+
+def parser():
+    cli = raft_output.Parser(description=__doc__)
+    cli.add_argument(
+        "--json",
+        dest="json_output",
+        action="store_true",
+        help="Emit JSONL data, byte-stream, error and exit events; use before the command",
+    )
+    cli.add_argument("--version", action="version", version=__version__)
+    commands = cli.add_subparsers(dest="action", required=True)
+    complete = commands.add_parser("completions", help="Print parser-driven shell completion setup")
+    complete.add_argument("shell", choices=["bash", "zsh", "fish"])
+    query = commands.add_parser("_complete")
+    query.add_argument("words", nargs=argparse.REMAINDER)
+    create = commands.add_parser(
+        "new",
+        help="Create a workspace and print its location-qualified handle",
+        description="Create from the pinned image or a same-host named snapshot. "
+        "Prints the handle before guest services are necessarily ready. "
+        "Inspect config for defaults and required notes, and capacity for host resources. "
+        "Disposable boxes are deleted on stop or expiry; persistent disks remain subject to host retention.",
+    )
+    creation_options(create)
+    run = commands.add_parser("run", help="Run a command in a temporary CI box and clean up")
+    creation_options(run, lifecycle=False)
+    raft_run.add_options(run)
     commands.add_parser("locations", help="List configured locations and their notes")
     defaults = commands.add_parser(
         "config", help="Show defaults for new and each location's note policy, or change defaults"
     )
+    defaults.add_argument("--location", help="Read policy only from this configured host")
     default_commands = defaults.add_subparsers(dest="config_action")
     default_set = default_commands.add_parser("set", help="Set a default for new")
     default_set.add_argument("key", choices=list(NEW_DEFAULTS))
@@ -1731,12 +1992,20 @@ def parser():
     default_unset = default_commands.add_parser("unset", help="Restore a built-in default")
     default_unset.add_argument("key", choices=list(NEW_DEFAULTS))
     policy = commands.add_parser(
-        "policy", help="Set or clear the box note fields a location's host requires"
+        "policy",
+        help="Set a location's required box note fields and stopped-box retention",
     )
     policy.add_argument("location")
-    rule = policy.add_mutually_exclusive_group(required=True)
-    rule.add_argument("--require", metavar="FIELDS", help="Note fields, e.g. purpose,owner")
-    rule.add_argument("--clear", action="store_true", help="Stop requiring note fields")
+    policy.add_argument(
+        "--require",
+        metavar="FIELDS|none",
+        help="Note fields every box must carry, e.g. purpose,owner; none stops requiring them",
+    )
+    policy.add_argument(
+        "--delete-stopped-after",
+        metavar="HOURS|never",
+        help="Delete boxes stopped for HOURS, e.g. 48; never keeps them",
+    )
     note = commands.add_parser("note", help="Set or clear the note of a location or a box")
     note.add_argument("target", metavar="LOCATION|BOX", help="A location name or a box handle")
     note.add_argument("text", nargs="?", help="Location guidance, e.g. 'Preferred for builds'")
@@ -1771,13 +2040,20 @@ def parser():
         help="Enable own box disks: set aside SIZE of host disk for them, e.g. 40GiB "
         "(default: not enabled)",
     )
-    listing = commands.add_parser("list")
+    listing = commands.add_parser(
+        "list",
+        help="List boxes as JSONL, including ownership notes and retention deadlines",
+        description="List boxes as JSONL. Read note before using a box; deleteAt is its stopped-box deletion deadline, or empty when none is due.",
+    )
     listing.add_argument("--location")
     report = commands.add_parser("capacity", help=ADVISORY, description=ADVISORY)
     report.add_argument("--location")
     report.add_argument("--json", action="store_true")
-    commands.add_parser("doctor")
-    gc = commands.add_parser("gc", help="Enforce deadlines, including disposable deletion")
+    doctor_command = commands.add_parser("doctor", help="Check host services, worker and storage")
+    doctor_command.add_argument("--location", help="Inspect only this configured host")
+    gc = commands.add_parser(
+        "gc", help="Enforce deadlines and retention, including disposable deletion"
+    )
     gc.add_argument("--location")
     prune = commands.add_parser("prune", help="Preview deletion of old stopped boxes")
     prune.add_argument("--location")
@@ -1802,33 +2078,38 @@ def parser():
         metavar="X.Y.Z",
         help="Install this release instead of the latest, e.g. to roll back",
     )
-    for action in [
-        "info",
-        "stop",
-        "resume",
-        "extend",
-        "destroy",
-        "ssh",
-        "exec",
-        "upload",
-        "download",
-        "snapshot",
-        "snapshots",
-        "snapshot-delete",
-        "restore",
-        "fork",
-        "forward",
-        "desktop",
-        "logs",
-        "status",
-        "cancel",
-        "backup",
-        "usage",
-    ]:
-        command = commands.add_parser(action)
-        command.add_argument("box")
+    for action, description in {
+        "info": "Read native box metadata as JSON without starting guest processes.",
+        "stop": "Stop processes in a persistent box and retain its disk. Disposable boxes and their snapshots are deleted. Inspect info before stopping.",
+        "resume": "Start a stopped persistent box with a new lifetime. Optional CPU and memory replace only those limits. Process memory is not restored.",
+        "extend": "Reset the deadline of a running box from the host clock without restarting it. Use resume for stopped boxes.",
+        "destroy": "Delete a box and all its snapshots, then confirm absence. Only destroy boxes owned by this task unless explicitly authorized.",
+        "ssh": "Open a root shell through host SSH and Incus exec. This does not expose a guest SSH port.",
+        "exec": "Run literal command arguments in /workspace and retain the exit status. Use an explicit shell for pipelines or expansions. Detached execution returns a job ID for status, logs and cancel.",
+        "sync": "Incrementally sync source into a running guest. Matching files are replaced; guest-only files remain. Requires rsync on controller and guest. Git selects tracked and unignored files by default; --all includes ignored files. Unsafe symlinks and special files are skipped by rsync. Updates are atomic per file, not per tree.",
+        "upload": "Copy a local file to an absolute guest path. Recursive transfers require a running guest and publish a new directory without merging or overwriting; escaping links and special files are rejected.",
+        "download": "Copy an absolute guest path to a local file. Recursive transfers require a running guest and a new destination directory; the parent must exist. Failed downloads do not replace existing files.",
+        "snapshot": "Save a named disk snapshot of a stopped persistent box. Snapshots disappear when their box is deleted, including by retention.",
+        "snapshots": "List named snapshots through native Incus info.",
+        "snapshot-delete": "Delete one named snapshot permanently without deleting its box.",
+        "restore": "Replace disk contents from a named snapshot of a stopped box. Keep current instance configuration, resource limits and MAC address. Requires an explicitly requested rollback.",
+        "fork": "Copy a stopped box on the same host, including files, secrets and note. Defaults to persistent. CPU and memory retain source limits unless overridden. Update the copy note for its new task.",
+        "forward": "Open a foreground SSH tunnel bound to controller 127.0.0.1. Guest service must listen on its managed NIC or 0.0.0.0. Ctrl+C closes the tunnel.",
+        "desktop": "Start the full guest desktop and open its private foreground tunnel. Closing the tunnel leaves the desktop service running until explicitly stopped or the box stops.",
+        "logs": "Print retained detached-job output, including after workspace stop or deletion. First 1 MiB retained for seven days.",
+        "status": "Print retained job state, exit code, timestamps and dropped bytes as JSON, even after workspace deletion.",
+        "cancel": "Stop the guest job control group. Host-retained output remains available for seven days.",
+        "backup": "Export a stopped persistent box and its snapshots to a new private archive. Includes credentials. The host lifecycle lock delays expiry while exporting.",
+        "usage": "Report running guest cgroup CPU and memory counters and filesystem space as JSON. filesystemScope distinguishes shared storage from an own disk.",
+    }.items():
+        command = commands.add_parser(
+            action, help=description.split(". ")[0], description=description
+        )
+        command.add_argument("box", help="Exact LOCATION:BOX handle returned by creation")
         if action in ["resume", "fork", "extend"]:
-            command.add_argument("--ttl", type=ttl, required=True)
+            command.add_argument(
+                "--ttl", type=ttl, required=True, help="Lifetime in seconds, 60 to 2592000"
+            )
         if action in ["resume", "fork"]:
             command.add_argument("--cpu", type=cpu_count)
             command.add_argument("--memory", type=gib_size)
@@ -1838,6 +2119,17 @@ def parser():
         if action == "exec":
             command.add_argument("--detach", action="store_true")
             command.add_argument("command", nargs="+")
+        if action == "sync":
+            command.add_argument("source")
+            command.add_argument("destination")
+            command.add_argument(
+                "--all",
+                action="store_true",
+                help="Include ignored and hidden files; inspect for secrets first",
+            )
+            command.add_argument(
+                "--dry-run", action="store_true", help="Report changes without writing guest files"
+            )
         if action in ["upload", "download"]:
             command.add_argument("source")
             command.add_argument("destination")
@@ -1857,6 +2149,13 @@ def parser():
 
 
 def dispatch(args):
+    if getattr(args, "json_output", False) and args.action in {"ssh", "forward", "desktop"}:
+        raise ValueError("JSON mode does not support interactive shells or foreground tunnels")
+    if args.action == "completions":
+        print(raft_output.completions(args.shell), end="")
+        return 0
+    if args.action == "run":
+        return raft_run.run(args, sys.modules[__name__])
     if args.action == "self-update":
         return self_update(args)
     if args.action == "recover":
@@ -1871,17 +2170,22 @@ def dispatch(args):
     if args.action == "config":
         if args.config_action is None:
             report = {"file": str(DEFAULTS_CONFIG), "defaults": effective_defaults(), "policy": {}}
-            for location in configuration():
+            locations = [args.location] if args.location else configuration()
+            for location in locations:
                 try:
-                    report["policy"][location] = note_policy(location)
+                    report["policy"][location] = host_policy(location)
                 except RuntimeError as error:
                     raise RuntimeError(f"{location}: {error}") from error
-            print(json.dumps(report, indent=2))
+            raft_output.data(report, indent=2)
         else:
+            if args.location:
+                raise ValueError(
+                    "--location selects a host policy report; defaults are controller-wide"
+                )
             set_default(args.key, args.value if args.config_action == "set" else None)
         return 0
     if args.action == "policy":
-        set_policy(args.location, None if args.clear else note_fields(args.require))
+        change_policy(args)
         return 0
     if args.action == "note":
         fields = box_note(args)
@@ -1919,7 +2223,7 @@ def dispatch(args):
     if args.action == "locations":
         for index, (location, entry) in enumerate(configuration().items()):
             record = {"location": location, "default": index == 0, "note": entry.get("note", "")}
-            print(json.dumps(record))
+            raft_output.data(record)
         return 0
     if args.action in ["list", "capacity", "doctor", "gc", "prune"]:
         if args.action == "prune" and args.older_than < 1:
@@ -1930,8 +2234,8 @@ def dispatch(args):
         for location in locations:
             if args.action == "capacity":
                 report = host_capacity(location)
-                if args.json:
-                    print(json.dumps(report))
+                if args.json or args.json_output:
+                    raft_output.data(report)
                 else:
                     print_capacity(report)
             elif args.action in ["gc", "prune"]:
@@ -1966,31 +2270,70 @@ def dispatch(args):
                     for line in output.splitlines():
                         record = json.loads(line)
                         record["box"] = location + ":" + record["box"]
-                        print(json.dumps(record))
+                        raft_output.data(record)
                 else:
                     print(output, end="")
             elif args.action == "doctor":
                 doctor(location)
             else:
-                for box in inventory(location):
-                    print(
-                        json.dumps(
-                            {
-                                "box": location + ":" + box["name"],
-                                "state": box["status"],
-                                "stopAt": box["config"].get("user.raft.expires", ""),
-                                "disk": box_disk(box),
-                                "note": read_box_note(box),
-                            }
-                        )
+                policy, boxes = host_inventory(location)
+                for box in boxes:
+                    deletion = delete_at(box, policy)
+                    raft_output.data(
+                        {
+                            "box": location + ":" + box["name"],
+                            "state": box["status"],
+                            "stopAt": box["config"].get("user.raft.expires", ""),
+                            "deleteAt": "" if deletion is None else str(deletion),
+                            "disk": box_disk(box),
+                            "note": read_box_note(box),
+                        }
                     )
         return 0
+    if args.action in {"logs", "status", "cancel"}:
+        location, name = parse_handle(args.box)
+        if not re.fullmatch(r"rfcmd-[a-f0-9]{32}", args.job):
+            raise ValueError("Use the job ID returned by exec --detach")
+        command = shlex.join(
+            [
+                "sudo",
+                "-n",
+                "python3",
+                "-c",
+                Path(raft_jobs.__file__).read_text(),
+                args.action,
+                name,
+                args.job,
+            ]
+        )
+        response = subprocess.run(
+            [
+                "ssh",
+                "-T",
+                "-o",
+                "BatchMode=yes",
+                "-o",
+                "ConnectTimeout=10",
+                settings(location)["ssh"],
+                command,
+            ],
+            capture_output=True,
+        )
+        if response.returncode:
+            raise RuntimeError(response.stderr.decode(errors="replace").strip())
+        if args.action == "status":
+            record = json.loads(response.stdout)
+            record["box"] = args.box
+            print(json.dumps(record))
+        else:
+            sys.stdout.buffer.write(response.stdout)
+        return 0
     # Only fork accepts note flags here; check them before contacting the host.
     fields = box_note(args)
     location, name = parse_handle(args.box)
     box = instance(location, name)
     if args.action == "info":
-        print(json.dumps(box, indent=2))
+        raft_output.data(box, indent=2)
     elif args.action == "backup":
         backup(args, location, name)
     elif args.action == "usage":
@@ -2003,7 +2346,7 @@ def dispatch(args):
         usage["filesystemScope"] = (
             "shared Raft pool, not per-box usage" if disk == "shared" else f"own {disk} box disk"
         )
-        print(json.dumps(usage))
+        raft_output.data(usage)
     elif args.action == "destroy":
         incus(location, "delete", name, "--force", locked=True)
         if any(item["name"] == name for item in inventory(location)):
@@ -2060,26 +2403,21 @@ def dispatch(args):
             raise ValueError("Supply a command after --")
         if args.detach:
             job = "rfcmd-" + uuid.uuid4().hex
-            response = execute(
-                location,
-                name,
-                [
-                    "systemd-run",
-                    "--quiet",
-                    "--unit",
+            source = Path(raft_jobs.__file__).read_text()
+            print(
+                transaction(
+                    location,
+                    'exec python3 -c "$1" start "$2" "$3" "$4" "$1"',
+                    source,
+                    name,
                     job,
-                    "--property=WorkingDirectory=/workspace",
-                    "--property=RemainAfterExit=yes",
-                    "--",
-                    *payload,
-                ],
-                capture=True,
+                    json.dumps(payload),
+                ).strip()
             )
-            if response.returncode:
-                raise RuntimeError(response.stderr.decode().strip())
-            print(job)
         else:
             return execute(location, name, payload).returncode
+    elif args.action == "sync":
+        return sync_source(args, location, name)
     elif args.action in ["upload", "download"]:
         guest_path = args.destination if args.action == "upload" else args.source
         if not guest_path.startswith("/"):
@@ -2154,44 +2492,6 @@ def dispatch(args):
             *sizing(args),
         )
         print(location + ":" + child)
-    elif args.action in ["logs", "status", "cancel"]:
-        if not re.fullmatch(r"rfcmd-[a-f0-9]{32}", args.job):
-            raise ValueError("Use the job ID returned by exec --detach")
-        loaded = execute(
-            location,
-            name,
-            ["systemctl", "show", args.job, "-p", "LoadState", "--value"],
-            capture=True,
-        )
-        if loaded.returncode or loaded.stdout.strip() != b"loaded":
-            # Cancelled transient units unload, but their journal keeps history.
-            # Live units remain recognizable even after old logs are rotated away.
-            recorded = execute(
-                location,
-                name,
-                ["journalctl", "--no-pager", "--quiet", "-o", "json", "-n", "1", "-u", args.job],
-                capture=True,
-            )
-            if recorded.returncode or not recorded.stdout.strip():
-                raise ValueError("Detached job not found in this running box")
-        if args.action == "cancel":
-            return execute(location, name, ["systemctl", "stop", args.job]).returncode
-        command = (
-            ["journalctl", "--no-pager", "-o", "cat", "-u", args.job]
-            if args.action == "logs"
-            else [
-                "systemctl",
-                "show",
-                args.job,
-                "-p",
-                "ActiveState",
-                "-p",
-                "SubState",
-                "-p",
-                "ExecMainStatus",
-            ]
-        )
-        return execute(location, name, command).returncode
     elif args.action in ["forward", "desktop"]:
         port = args.remote if args.action == "forward" else 6080
         if not 1 <= port <= 65535 or not 1 <= args.local <= 65535:
@@ -2204,16 +2504,42 @@ def dispatch(args):
     return 0
 
 
+def exit_status(returncode):
+    """Translate Python subprocess signals into shell-compatible process statuses."""
+    return 128 - returncode if returncode < 0 else returncode
+
+
 def main():
     if sys.version_info < (3, 11, 8):
         print("raft: Python 3.11.8 or newer is required", file=sys.stderr)
         return 1
     os.umask(0o077)
-    # Parsing exits for --version, --help and usage errors, before any release check.
-    args = parser().parse_args()
-    notice = update_check(args.action, RELEASES_API)
+    # Only root options select this mode. A payload's --json is its own argument.
+    root_options = []
+    for argument in sys.argv[1:]:
+        if not argument.startswith("-") or argument == "--":
+            break
+        root_options.append(argument)
+    output = raft_output.JsonOutput().start() if "--json" in root_options else None
+    status = 1
+    notice = None
     try:
+        cli = parser()
+        args = cli.parse_args()
+        if args.action == "_complete":
+            words = args.words[1:] if args.words[:1] == ["--"] else args.words
+            print("\n".join(raft_output.candidates(cli, words)))
+            status = 0
+            return status
+        notice = (
+            None
+            if output or args.action in {"completions", "_complete"}
+            else update_check(args.action, RELEASES_API)
+        )
         status = dispatch(args)
+    except raft_output.UsageError as error:
+        raft_output.error("usage", error)
+        status = 2
     except (
         OSError,
         ValueError,
@@ -2221,11 +2547,18 @@ def main():
         tarfile.TarError,
         subprocess.CalledProcessError,
     ) as error:
-        print(f"raft: {error}", file=sys.stderr)
+        raft_output.error("operation", error)
         status = 1
+    except SystemExit as error:
+        status = error.code if isinstance(error.code, int) else 1
     except KeyboardInterrupt:
-        return 130
-    notice()
+        status = 130
+    finally:
+        status = exit_status(status)
+        if notice and status != 130:
+            notice()
+        if output:
+            output.finish(status)
     return status