@microck/raft 0.6.0 → 0.7.0

package/raft.py

@@ -24,8 +24,11 @@ import urllib.request
 import uuid
 
 import raft_files
+import raft_run
+import raft_jobs
+import raft_output
 
-__version__ = "0.6.0"
+__version__ = "0.7.0"
 
 CONFIG = Path.home() / ".config/raft/incus.json"
 # Controller defaults for new live apart from the location inventory, so location
@@ -230,6 +233,11 @@ def host_policy(location):
     # `incus query` rejects the --project flag that remote() always passes.
     projects = json.loads(incus(location, "project", "list", "--format", "json"))
     config = next(item for item in projects if item["name"] == PROJECT)["config"]
+    return parse_host_policy(location, config)
+
+
+def parse_host_policy(location, config):
+    """Validate native policy values before displaying or enforcing them."""
     stored = config.get(POLICY_KEY, "")
     required = [field for field in stored.split(",") if field]
     unknown = [field for field in required if field not in BOX_NOTE_FIELDS]
@@ -381,11 +389,11 @@ def receive(command, destination, *, stdin=None):
         ) as process:
             try:
                 shutil.copyfileobj(process.stdout, destination)
+                returncode = process.wait()
             except BaseException:
                 process.kill()
                 process.wait()
                 raise
-            returncode = process.wait()
         errors.seek(0)
         return subprocess.CompletedProcess(command, returncode, None, errors.read())
 
@@ -654,6 +662,30 @@ def inventory(location):
     ]
 
 
+# These probes retain --fast so stopped/stopping guests never need live counters.
+HOST_BOXES = (
+    "project = "
+    + repr(PROJECT)
+    + "\n"
+    + """
+print(json.dumps(json.loads(subprocess.run(['incus', '--project', project, 'list', '--fast', '--format', 'json'],
+                     check=True, capture_output=True).stdout)))
+"""
+)
+HOST_INVENTORY = (
+    "project = "
+    + repr(PROJECT)
+    + "\n"
+    + """import json, subprocess
+projects = json.loads(subprocess.run(['incus', '--project', project, 'project', 'list', '--format', 'json'],
+                                    check=True, capture_output=True).stdout)
+boxes = json.loads(subprocess.run(['incus', '--project', project, 'list', '--fast', '--format', 'json'],
+                                 check=True, capture_output=True).stdout)
+print(json.dumps({'policy': next(p for p in projects if p['name'] == project)['config'], 'boxes': boxes}))
+"""
+)
+
+
 REQUIRE_STOPPED = """state=$(incus --project raft list "$1" --fast --format json | python3 -c 'import json,sys; print(next(x["status"] for x in json.load(sys.stdin) if x["name"] == sys.argv[1] and x["config"].get("user.raft") == "true"))' "$1")
 test "$state" = Stopped || { echo 'Stop the source box before resume, snapshot, restore, fork or backup' >&2; exit 1; }
 """
@@ -917,26 +949,44 @@ def capacity(host, boxes):
     }
 
 
-def host_capacity(location):
+def host_read(location, script):
+    """Run a read-only host probe without taking the lifecycle/expiry lock."""
     response = subprocess.run(
         [
             "ssh",
+            "-n",
             "-T",
             "-o",
             "BatchMode=yes",
             "-o",
             "ConnectTimeout=10",
             settings(location)["ssh"],
-            shlex.join(["sudo", "-n", "python3", "-c", HOST_RESOURCES]),
+            shlex.join(["sudo", "-n", "python3", "-c", script]),
         ],
         capture_output=True,
         text=True,
-        check=True,
     )
+    if response.returncode:
+        raise RuntimeError(f"{location}: {response.stderr.strip()}")
+    return response.stdout
+
+
+def host_inventory(location):
+    # Keep both native reads on the host: network latency costs more than JSON
+    # decoding. Do not lock read-only inventory behind a long backup or creation.
+    report = json.loads(host_read(location, HOST_INVENTORY))
+    policy = parse_host_policy(location, report["policy"])
+    boxes = [box for box in report["boxes"] if box["config"].get("user.raft") == "true"]
+    return policy, boxes
+
+
+def host_capacity(location):
+    resources, boxes = host_read(location, HOST_RESOURCES + HOST_BOXES).splitlines()
+    boxes = [box for box in json.loads(boxes) if box["config"].get("user.raft") == "true"]
     return {
         "location": location,
         "note": settings(location).get("note", ""),
-        **capacity(json.loads(response.stdout), inventory(location)),
+        **capacity(json.loads(resources), boxes),
     }
 
 
@@ -986,34 +1036,34 @@ def print_capacity(report):
 
 
 def doctor(location):
-    print(location + ": " + incus(location, "version").strip())
-    try:
-        transaction(location, CHECK_WORKER)
-    except RuntimeError as error:
-        raise RuntimeError(f"{location}: {error}") from error
-    response = subprocess.run(
-        [
-            "ssh",
-            "-T",
-            "-o",
-            "BatchMode=yes",
-            "-o",
-            "ConnectTimeout=10",
-            settings(location)["ssh"],
-            "systemctl is-active incus raft-expire.timer raft-network.service && df -h /",
-        ],
-        capture_output=True,
-        text=True,
-        check=True,
+    # Health checks are sequential on the host, not separate SSH handshakes.
+    # Only the worker compatibility check takes the lifecycle lock, as before.
+    script = "import json, subprocess, sys\nworker_check = " + repr(CHECK_WORKER) + "\n"
+    script += (
+        f"project = {PROJECT!r}\nshared_pool = {SHARED_POOL!r}\nbox_disk_pool = {BOX_DISK_POOL!r}\n"
     )
-    print(response.stdout.strip())
-    print(incus(location, "storage", "info", SHARED_POOL).strip())
-    # Own box disks are opt-in, so a host without their pool is healthy.
-    if BOX_DISK_POOL in incus(location, "storage", "list", "--format", "csv", "-c", "n").split():
-        print(incus(location, "storage", "info", BOX_DISK_POOL).strip())
-    else:
-        print("own box disks: not enabled (redeploy with --disk-pool SIZE)")
-    print(f"{len(inventory(location))} saved boxes; system containers (shared host kernel)")
+    script += """
+def checked(*command):
+    response = subprocess.run(command, capture_output=True, text=True)
+    if response.returncode:
+        sys.exit(response.stderr.strip() or response.stdout.strip())
+    return response.stdout.strip()
+print(checked('incus', '--project', project, 'version'))
+checked('flock', '/run/lock/raft-incus.lock', 'sh', '-ec', worker_check)
+print(checked('systemctl', 'is-active', 'incus', 'raft-expire.timer', 'raft-network.service'))
+print(checked('df', '-h', '/'))
+print(checked('incus', '--project', project, 'storage', 'info', shared_pool))
+pools = checked('incus', '--project', project, 'storage', 'list', '--format', 'csv', '-c', 'n').split()
+if box_disk_pool in pools:
+    print(checked('incus', '--project', project, 'storage', 'info', box_disk_pool))
+else:
+    print('own box disks: not enabled (redeploy with --disk-pool SIZE)')
+boxes = json.loads(checked('incus', '--project', project, 'list', '--fast', '--format', 'json'))
+print(str(sum(box['config'].get('user.raft') == 'true' for box in boxes))
+      + ' saved boxes; system containers (shared host kernel)')
+"""
+    print(location + ": ", end="", flush=True)
+    print(host_read(location, script), end="")
 
 
 def transaction(location, script, *arguments, stdout=None):
@@ -1048,6 +1098,57 @@ def transaction(location, script, *arguments, stdout=None):
     return response.stdout.decode() if response.stdout is not None else ""
 
 
+def sync_source(args, location, name):
+    """Use rsync's checksum/delta protocol inside the guest, never on the host tree."""
+    if shutil.which("rsync") is None:
+        raise ValueError("sync requires rsync on the controller; install rsync first")
+    source = Path(args.source).resolve(strict=True)
+    if not source.is_dir():
+        raise ValueError("Sync source must be a directory")
+    destination = raft_run.guest_path(args.destination)
+    if destination == "/":
+        raise ValueError("Sync destination must not be the guest root directory")
+    selected = None if args.all else raft_run.git_files(source, all_option="--all")
+    filters = None
+    if selected is not None:
+        # Recursive traversal never follows directory symlinks. A files-from list
+        # would follow symlink ancestors when a tracked directory was replaced.
+        allowed = set(selected)
+        for path in selected:
+            allowed.update(str(parent) for parent in Path(path).parents if str(parent) != ".")
+        rules = []
+        for path in sorted(allowed):
+            if any(char in path for char in "*?["):
+                path = re.sub(r"([\\*?\[])", r"\\\1", path)
+            rules.append(b"+ /" + os.fsencode(path))
+        filters = b"\0".join([*rules, b"- *", b""])
+    receiver = shlex.join(
+        ["sudo", "-n", "incus", "--project", PROJECT, "exec", name, "--", "rsync"]
+    )
+    command = [
+        "rsync",
+        "--recursive",
+        "--links",
+        "--safe-links",
+        "--times",
+        "--perms",
+        "--checksum",
+        "--omit-dir-times",
+        "--itemize-changes",
+        "--secluded-args",
+        "--rsync-path",
+        receiver,
+        "-e",
+        "ssh -T -o BatchMode=yes -o ConnectTimeout=10",
+    ]
+    if filters is not None:
+        command += ["--from0", "--filter=merge -"]
+    if args.dry_run:
+        command.append("--dry-run")
+    command += ["--", str(source) + "/", settings(location)["ssh"] + ":" + destination + "/"]
+    return subprocess.run(command, input=filters).returncode
+
+
 def snapshot_name(value):
     if not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_-]{0,63}", value):
         raise ValueError("Invalid snapshot name")
@@ -1104,7 +1205,7 @@ def own_disk(size):
     ]
 
 
-def new(args):
+def new(args, *, run_id=None, on_allocate=None):
     box_fields = box_note(args)
     # Sizes are filled here only for boxes from the image; a box from a snapshot
     # keeps the snapshot's CPU, RAM and disk unless a flag overrides them.
@@ -1133,10 +1234,13 @@ def new(args):
         print(f"{args.location} note: {note}", file=sys.stderr)
     require_note(args.location, box_fields, "new")
     name = "rf-" + uuid.uuid4().hex[:16]
+    if on_allocate is not None:
+        on_allocate(args.location, name)
     # Initialize before starting, so a failed start still has its intended policy.
     transaction(
         args.location,
-        (CHECK_WORKER if args.disposable else "")
+        (raft_run.creation_guard(run_id) if run_id else "")
+        + (CHECK_WORKER if args.disposable else "")
         + fits_host(args)
         # Plain boxes from the image stay off raft-disks and skip the check.
         + (
@@ -1155,10 +1259,15 @@ def new(args):
         + CHECK_NOTE
         # Always set the note, so a box created from a snapshot never inherits the
         # source note. The host checks the same value it writes.
-        + '\n"$@" -c user.raft.note="$note"; '
-        'incus --project raft start "$name"; '
-        'incus --project raft config set "$name" '
-        'user.raft.expires="$(( $(date +%s) + lifetime ))"',
+        + '\n"$@" -c user.raft.note="$note"'
+        + (' -c user.raft.expires="$(( $(date +%s) + lifetime ))"' if run_id else "")
+        + '; incus --project raft start "$name"; '
+        + (
+            ""
+            if run_id
+            else 'incus --project raft config set "$name" '
+            'user.raft.expires="$(( $(date +%s) + lifetime ))"'
+        ),
         str(args.ttl),
         name,
         json.dumps(box_fields) if box_fields else "",
@@ -1170,8 +1279,8 @@ def new(args):
         name,
         "-c",
         "user.raft=true",
-        "-c",
-        "user.raft.expires=",
+        *([] if run_id else ["-c", "user.raft.expires="]),
+        *(["-c", f"user.raft.run={run_id}"] if run_id else []),
         "-c",
         "user.raft.stopped-at=",
         "-c",
@@ -1187,15 +1296,15 @@ def new(args):
     return f"{args.location}:{name}"
 
 
-def execute(location, name, command, *, capture=False):
-    return remote(location, ["exec", name, "--cwd", "/workspace", "--", *command], capture=capture)
+def execute(location, name, command, *, capture=False, cwd="/workspace"):
+    return remote(location, ["exec", name, "--cwd", cwd, "--", *command], capture=capture)
 
 
 def transfer_tree(args, location, name):
     worker = Path(raft_files.__file__).read_text()
     with tempfile.TemporaryFile() as archive:
         if args.action == "upload":
-            raft_files.pack_directory(args.source, archive)
+            raft_files.pack_directory(args.source, archive, include=getattr(args, "include", None))
             size = archive.tell()
             archive.seek(0)
             response = remote(
@@ -1793,27 +1902,26 @@ def note_options(command):
     )
 
 
-def parser():
-    cli = argparse.ArgumentParser(description=__doc__)
-    cli.add_argument("--version", action="version", version=__version__)
-    commands = cli.add_subparsers(dest="action", required=True)
-    create = commands.add_parser("new")
-    create.add_argument("--location", default=None)
+def creation_options(command, *, lifecycle=True):
+    command.add_argument("--location", default=None)
     # Omitted options take ~/.config/raft/config.json defaults, then NEW_DEFAULTS.
     # With --from, every omitted size comes from the snapshot instead of a default.
-    create.add_argument("--ttl", type=ttl)
-    create.add_argument(
+    if lifecycle:
+        command.add_argument(
+            "--ttl", type=ttl, help="Lifetime in seconds, 60 to 2592000 (default: config, else 600)"
+        )
+    command.add_argument(
         "--cpu",
         type=cpu_count,
         help=f"CPU limit (default: config, else {DEFAULT_CPU}; with --from, the snapshot's)",
     )
-    create.add_argument(
+    command.add_argument(
         "--memory",
         type=gib_size,
         help="RAM limit, e.g. 16GiB "
         f"(default: config, else {DEFAULT_MEMORY}; with --from, the snapshot's)",
     )
-    create.add_argument(
+    command.add_argument(
         "--disk",
         type=disk_size,
         metavar="SIZE",
@@ -1821,27 +1929,62 @@ def parser():
         "or shared for the shared pool "
         "(default: config, else shared; with --from, the snapshot's disk)",
     )
-    lifecycle = create.add_mutually_exclusive_group()
-    lifecycle.add_argument(
-        "--disposable",
-        dest="disposable",
-        action="store_const",
-        const=True,
-        help="Delete on stop or expiry (default: config, else persistent)",
+    if lifecycle:
+        policy = command.add_mutually_exclusive_group()
+        policy.add_argument(
+            "--disposable",
+            dest="disposable",
+            action="store_const",
+            const=True,
+            help="Delete on stop or expiry (default: config, else persistent)",
+        )
+        policy.add_argument(
+            "--persistent",
+            dest="disposable",
+            action="store_const",
+            const=False,
+            help="Keep the disk on stop or expiry, despite a disposable default",
+        )
+    command.add_argument(
+        "--from",
+        dest="template",
+        metavar="LOCATION:BOX/SNAPSHOT",
+        help="Copy a named snapshot on its source host, including filesystem secrets",
     )
-    lifecycle.add_argument(
-        "--persistent",
-        dest="disposable",
-        action="store_const",
-        const=False,
-        help="Keep the disk on stop or expiry, despite a disposable default",
+    note_options(command)
+
+
+def parser():
+    cli = raft_output.Parser(description=__doc__)
+    cli.add_argument(
+        "--json",
+        dest="json_output",
+        action="store_true",
+        help="Emit JSONL data, byte-stream, error and exit events; use before the command",
+    )
+    cli.add_argument("--version", action="version", version=__version__)
+    commands = cli.add_subparsers(dest="action", required=True)
+    complete = commands.add_parser("completions", help="Print parser-driven shell completion setup")
+    complete.add_argument("shell", choices=["bash", "zsh", "fish"])
+    query = commands.add_parser("_complete")
+    query.add_argument("words", nargs=argparse.REMAINDER)
+    create = commands.add_parser(
+        "new",
+        help="Create a workspace and print its location-qualified handle",
+        description="Create from the pinned image or a same-host named snapshot. "
+        "Prints the handle before guest services are necessarily ready. "
+        "Inspect config for defaults and required notes, and capacity for host resources. "
+        "Disposable boxes are deleted on stop or expiry; persistent disks remain subject to host retention.",
     )
-    create.add_argument("--from", dest="template", metavar="LOCATION:BOX/SNAPSHOT")
-    note_options(create)
+    creation_options(create)
+    run = commands.add_parser("run", help="Run a command in a temporary CI box and clean up")
+    creation_options(run, lifecycle=False)
+    raft_run.add_options(run)
     commands.add_parser("locations", help="List configured locations and their notes")
     defaults = commands.add_parser(
         "config", help="Show defaults for new and each location's note policy, or change defaults"
     )
+    defaults.add_argument("--location", help="Read policy only from this configured host")
     default_commands = defaults.add_subparsers(dest="config_action")
     default_set = default_commands.add_parser("set", help="Set a default for new")
     default_set.add_argument("key", choices=list(NEW_DEFAULTS))
@@ -1897,12 +2040,17 @@ def parser():
         help="Enable own box disks: set aside SIZE of host disk for them, e.g. 40GiB "
         "(default: not enabled)",
     )
-    listing = commands.add_parser("list")
+    listing = commands.add_parser(
+        "list",
+        help="List boxes as JSONL, including ownership notes and retention deadlines",
+        description="List boxes as JSONL. Read note before using a box; deleteAt is its stopped-box deletion deadline, or empty when none is due.",
+    )
     listing.add_argument("--location")
     report = commands.add_parser("capacity", help=ADVISORY, description=ADVISORY)
     report.add_argument("--location")
     report.add_argument("--json", action="store_true")
-    commands.add_parser("doctor")
+    doctor_command = commands.add_parser("doctor", help="Check host services, worker and storage")
+    doctor_command.add_argument("--location", help="Inspect only this configured host")
     gc = commands.add_parser(
         "gc", help="Enforce deadlines and retention, including disposable deletion"
     )
@@ -1930,33 +2078,38 @@ def parser():
         metavar="X.Y.Z",
         help="Install this release instead of the latest, e.g. to roll back",
     )
-    for action in [
-        "info",
-        "stop",
-        "resume",
-        "extend",
-        "destroy",
-        "ssh",
-        "exec",
-        "upload",
-        "download",
-        "snapshot",
-        "snapshots",
-        "snapshot-delete",
-        "restore",
-        "fork",
-        "forward",
-        "desktop",
-        "logs",
-        "status",
-        "cancel",
-        "backup",
-        "usage",
-    ]:
-        command = commands.add_parser(action)
-        command.add_argument("box")
+    for action, description in {
+        "info": "Read native box metadata as JSON without starting guest processes.",
+        "stop": "Stop processes in a persistent box and retain its disk. Disposable boxes and their snapshots are deleted. Inspect info before stopping.",
+        "resume": "Start a stopped persistent box with a new lifetime. Optional CPU and memory replace only those limits. Process memory is not restored.",
+        "extend": "Reset the deadline of a running box from the host clock without restarting it. Use resume for stopped boxes.",
+        "destroy": "Delete a box and all its snapshots, then confirm absence. Only destroy boxes owned by this task unless explicitly authorized.",
+        "ssh": "Open a root shell through host SSH and Incus exec. This does not expose a guest SSH port.",
+        "exec": "Run literal command arguments in /workspace and retain the exit status. Use an explicit shell for pipelines or expansions. Detached execution returns a job ID for status, logs and cancel.",
+        "sync": "Incrementally sync source into a running guest. Matching files are replaced; guest-only files remain. Requires rsync on controller and guest. Git selects tracked and unignored files by default; --all includes ignored files. Unsafe symlinks and special files are skipped by rsync. Updates are atomic per file, not per tree.",
+        "upload": "Copy a local file to an absolute guest path. Recursive transfers require a running guest and publish a new directory without merging or overwriting; escaping links and special files are rejected.",
+        "download": "Copy an absolute guest path to a local file. Recursive transfers require a running guest and a new destination directory; the parent must exist. Failed downloads do not replace existing files.",
+        "snapshot": "Save a named disk snapshot of a stopped persistent box. Snapshots disappear when their box is deleted, including by retention.",
+        "snapshots": "List named snapshots through native Incus info.",
+        "snapshot-delete": "Delete one named snapshot permanently without deleting its box.",
+        "restore": "Replace disk contents from a named snapshot of a stopped box. Keep current instance configuration, resource limits and MAC address. Requires an explicitly requested rollback.",
+        "fork": "Copy a stopped box on the same host, including files, secrets and note. Defaults to persistent. CPU and memory retain source limits unless overridden. Update the copy note for its new task.",
+        "forward": "Open a foreground SSH tunnel bound to controller 127.0.0.1. Guest service must listen on its managed NIC or 0.0.0.0. Ctrl+C closes the tunnel.",
+        "desktop": "Start the full guest desktop and open its private foreground tunnel. Closing the tunnel leaves the desktop service running until explicitly stopped or the box stops.",
+        "logs": "Print retained detached-job output, including after workspace stop or deletion. First 1 MiB retained for seven days.",
+        "status": "Print retained job state, exit code, timestamps and dropped bytes as JSON, even after workspace deletion.",
+        "cancel": "Stop the guest job control group. Host-retained output remains available for seven days.",
+        "backup": "Export a stopped persistent box and its snapshots to a new private archive. Includes credentials. The host lifecycle lock delays expiry while exporting.",
+        "usage": "Report running guest cgroup CPU and memory counters and filesystem space as JSON. filesystemScope distinguishes shared storage from an own disk.",
+    }.items():
+        command = commands.add_parser(
+            action, help=description.split(". ")[0], description=description
+        )
+        command.add_argument("box", help="Exact LOCATION:BOX handle returned by creation")
         if action in ["resume", "fork", "extend"]:
-            command.add_argument("--ttl", type=ttl, required=True)
+            command.add_argument(
+                "--ttl", type=ttl, required=True, help="Lifetime in seconds, 60 to 2592000"
+            )
         if action in ["resume", "fork"]:
             command.add_argument("--cpu", type=cpu_count)
             command.add_argument("--memory", type=gib_size)
@@ -1966,6 +2119,17 @@ def parser():
         if action == "exec":
             command.add_argument("--detach", action="store_true")
             command.add_argument("command", nargs="+")
+        if action == "sync":
+            command.add_argument("source")
+            command.add_argument("destination")
+            command.add_argument(
+                "--all",
+                action="store_true",
+                help="Include ignored and hidden files; inspect for secrets first",
+            )
+            command.add_argument(
+                "--dry-run", action="store_true", help="Report changes without writing guest files"
+            )
         if action in ["upload", "download"]:
             command.add_argument("source")
             command.add_argument("destination")
@@ -1985,6 +2149,13 @@ def parser():
 
 
 def dispatch(args):
+    if getattr(args, "json_output", False) and args.action in {"ssh", "forward", "desktop"}:
+        raise ValueError("JSON mode does not support interactive shells or foreground tunnels")
+    if args.action == "completions":
+        print(raft_output.completions(args.shell), end="")
+        return 0
+    if args.action == "run":
+        return raft_run.run(args, sys.modules[__name__])
     if args.action == "self-update":
         return self_update(args)
     if args.action == "recover":
@@ -1999,13 +2170,18 @@ def dispatch(args):
     if args.action == "config":
         if args.config_action is None:
             report = {"file": str(DEFAULTS_CONFIG), "defaults": effective_defaults(), "policy": {}}
-            for location in configuration():
+            locations = [args.location] if args.location else configuration()
+            for location in locations:
                 try:
                     report["policy"][location] = host_policy(location)
                 except RuntimeError as error:
                     raise RuntimeError(f"{location}: {error}") from error
-            print(json.dumps(report, indent=2))
+            raft_output.data(report, indent=2)
         else:
+            if args.location:
+                raise ValueError(
+                    "--location selects a host policy report; defaults are controller-wide"
+                )
             set_default(args.key, args.value if args.config_action == "set" else None)
         return 0
     if args.action == "policy":
@@ -2047,7 +2223,7 @@ def dispatch(args):
     if args.action == "locations":
         for index, (location, entry) in enumerate(configuration().items()):
             record = {"location": location, "default": index == 0, "note": entry.get("note", "")}
-            print(json.dumps(record))
+            raft_output.data(record)
         return 0
     if args.action in ["list", "capacity", "doctor", "gc", "prune"]:
         if args.action == "prune" and args.older_than < 1:
@@ -2058,8 +2234,8 @@ def dispatch(args):
         for location in locations:
             if args.action == "capacity":
                 report = host_capacity(location)
-                if args.json:
-                    print(json.dumps(report))
+                if args.json or args.json_output:
+                    raft_output.data(report)
                 else:
                     print_capacity(report)
             elif args.action in ["gc", "prune"]:
@@ -2094,35 +2270,70 @@ def dispatch(args):
                     for line in output.splitlines():
                         record = json.loads(line)
                         record["box"] = location + ":" + record["box"]
-                        print(json.dumps(record))
+                        raft_output.data(record)
                 else:
                     print(output, end="")
             elif args.action == "doctor":
                 doctor(location)
             else:
-                policy = host_policy(location)
-                for box in inventory(location):
+                policy, boxes = host_inventory(location)
+                for box in boxes:
                     deletion = delete_at(box, policy)
-                    print(
-                        json.dumps(
-                            {
-                                "box": location + ":" + box["name"],
-                                "state": box["status"],
-                                "stopAt": box["config"].get("user.raft.expires", ""),
-                                # Epoch text like stopAt; empty when no retention deletion is due.
-                                "deleteAt": "" if deletion is None else str(deletion),
-                                "disk": box_disk(box),
-                                "note": read_box_note(box),
-                            }
-                        )
+                    raft_output.data(
+                        {
+                            "box": location + ":" + box["name"],
+                            "state": box["status"],
+                            "stopAt": box["config"].get("user.raft.expires", ""),
+                            "deleteAt": "" if deletion is None else str(deletion),
+                            "disk": box_disk(box),
+                            "note": read_box_note(box),
+                        }
                     )
         return 0
+    if args.action in {"logs", "status", "cancel"}:
+        location, name = parse_handle(args.box)
+        if not re.fullmatch(r"rfcmd-[a-f0-9]{32}", args.job):
+            raise ValueError("Use the job ID returned by exec --detach")
+        command = shlex.join(
+            [
+                "sudo",
+                "-n",
+                "python3",
+                "-c",
+                Path(raft_jobs.__file__).read_text(),
+                args.action,
+                name,
+                args.job,
+            ]
+        )
+        response = subprocess.run(
+            [
+                "ssh",
+                "-T",
+                "-o",
+                "BatchMode=yes",
+                "-o",
+                "ConnectTimeout=10",
+                settings(location)["ssh"],
+                command,
+            ],
+            capture_output=True,
+        )
+        if response.returncode:
+            raise RuntimeError(response.stderr.decode(errors="replace").strip())
+        if args.action == "status":
+            record = json.loads(response.stdout)
+            record["box"] = args.box
+            print(json.dumps(record))
+        else:
+            sys.stdout.buffer.write(response.stdout)
+        return 0
     # Only fork accepts note flags here; check them before contacting the host.
     fields = box_note(args)
     location, name = parse_handle(args.box)
     box = instance(location, name)
     if args.action == "info":
-        print(json.dumps(box, indent=2))
+        raft_output.data(box, indent=2)
     elif args.action == "backup":
         backup(args, location, name)
     elif args.action == "usage":
@@ -2135,7 +2346,7 @@ def dispatch(args):
         usage["filesystemScope"] = (
             "shared Raft pool, not per-box usage" if disk == "shared" else f"own {disk} box disk"
         )
-        print(json.dumps(usage))
+        raft_output.data(usage)
     elif args.action == "destroy":
         incus(location, "delete", name, "--force", locked=True)
         if any(item["name"] == name for item in inventory(location)):
@@ -2192,26 +2403,21 @@ def dispatch(args):
             raise ValueError("Supply a command after --")
         if args.detach:
             job = "rfcmd-" + uuid.uuid4().hex
-            response = execute(
-                location,
-                name,
-                [
-                    "systemd-run",
-                    "--quiet",
-                    "--unit",
+            source = Path(raft_jobs.__file__).read_text()
+            print(
+                transaction(
+                    location,
+                    'exec python3 -c "$1" start "$2" "$3" "$4" "$1"',
+                    source,
+                    name,
                     job,
-                    "--property=WorkingDirectory=/workspace",
-                    "--property=RemainAfterExit=yes",
-                    "--",
-                    *payload,
-                ],
-                capture=True,
+                    json.dumps(payload),
+                ).strip()
             )
-            if response.returncode:
-                raise RuntimeError(response.stderr.decode().strip())
-            print(job)
         else:
             return execute(location, name, payload).returncode
+    elif args.action == "sync":
+        return sync_source(args, location, name)
     elif args.action in ["upload", "download"]:
         guest_path = args.destination if args.action == "upload" else args.source
         if not guest_path.startswith("/"):
@@ -2286,44 +2492,6 @@ def dispatch(args):
             *sizing(args),
         )
         print(location + ":" + child)
-    elif args.action in ["logs", "status", "cancel"]:
-        if not re.fullmatch(r"rfcmd-[a-f0-9]{32}", args.job):
-            raise ValueError("Use the job ID returned by exec --detach")
-        loaded = execute(
-            location,
-            name,
-            ["systemctl", "show", args.job, "-p", "LoadState", "--value"],
-            capture=True,
-        )
-        if loaded.returncode or loaded.stdout.strip() != b"loaded":
-            # Cancelled transient units unload, but their journal keeps history.
-            # Live units remain recognizable even after old logs are rotated away.
-            recorded = execute(
-                location,
-                name,
-                ["journalctl", "--no-pager", "--quiet", "-o", "json", "-n", "1", "-u", args.job],
-                capture=True,
-            )
-            if recorded.returncode or not recorded.stdout.strip():
-                raise ValueError("Detached job not found in this running box")
-        if args.action == "cancel":
-            return execute(location, name, ["systemctl", "stop", args.job]).returncode
-        command = (
-            ["journalctl", "--no-pager", "-o", "cat", "-u", args.job]
-            if args.action == "logs"
-            else [
-                "systemctl",
-                "show",
-                args.job,
-                "-p",
-                "ActiveState",
-                "-p",
-                "SubState",
-                "-p",
-                "ExecMainStatus",
-            ]
-        )
-        return execute(location, name, command).returncode
     elif args.action in ["forward", "desktop"]:
         port = args.remote if args.action == "forward" else 6080
         if not 1 <= port <= 65535 or not 1 <= args.local <= 65535:
@@ -2336,16 +2504,42 @@ def dispatch(args):
     return 0
 
 
+def exit_status(returncode):
+    """Translate Python subprocess signals into shell-compatible process statuses."""
+    return 128 - returncode if returncode < 0 else returncode
+
+
 def main():
     if sys.version_info < (3, 11, 8):
         print("raft: Python 3.11.8 or newer is required", file=sys.stderr)
         return 1
     os.umask(0o077)
-    # Parsing exits for --version, --help and usage errors, before any release check.
-    args = parser().parse_args()
-    notice = update_check(args.action, RELEASES_API)
+    # Only root options select this mode. A payload's --json is its own argument.
+    root_options = []
+    for argument in sys.argv[1:]:
+        if not argument.startswith("-") or argument == "--":
+            break
+        root_options.append(argument)
+    output = raft_output.JsonOutput().start() if "--json" in root_options else None
+    status = 1
+    notice = None
     try:
+        cli = parser()
+        args = cli.parse_args()
+        if args.action == "_complete":
+            words = args.words[1:] if args.words[:1] == ["--"] else args.words
+            print("\n".join(raft_output.candidates(cli, words)))
+            status = 0
+            return status
+        notice = (
+            None
+            if output or args.action in {"completions", "_complete"}
+            else update_check(args.action, RELEASES_API)
+        )
         status = dispatch(args)
+    except raft_output.UsageError as error:
+        raft_output.error("usage", error)
+        status = 2
     except (
         OSError,
         ValueError,
@@ -2353,11 +2547,18 @@ def main():
         tarfile.TarError,
         subprocess.CalledProcessError,
     ) as error:
-        print(f"raft: {error}", file=sys.stderr)
+        raft_output.error("operation", error)
         status = 1
+    except SystemExit as error:
+        status = error.code if isinstance(error.code, int) else 1
     except KeyboardInterrupt:
-        return 130
-    notice()
+        status = 130
+    finally:
+        status = exit_status(status)
+        if notice and status != 130:
+            notice()
+        if output:
+            output.finish(status)
     return status