@microck/raft 0.0.0-stage → 0.2.0

package/raft_files.py

@@ -0,0 +1,134 @@
+"""Stream directory archives and publish new trees without replacing existing paths.
+
+The same code runs on the Linux controller and in the guest. Unpacking always
+uses Python's data filter, and publication uses renameat2(RENAME_NOREPLACE).
+"""
+
+import argparse
+import ctypes
+import os
+from pathlib import Path, PurePosixPath
+import shutil
+import stat
+import sys
+import tarfile
+import tempfile
+from types import SimpleNamespace
+
+
+def publish(source, destination):
+    libc = ctypes.CDLL(None, use_errno=True)
+    rename = libc.renameat2
+    rename.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint]
+    rename.restype = ctypes.c_int
+    if rename(-100, os.fsencode(source), -100, os.fsencode(destination), 1):
+        code = ctypes.get_errno()
+        raise OSError(code, os.strerror(code), str(destination))
+
+
+def safe_member(member, destination):
+    if PurePosixPath(member.name).is_absolute():
+        raise ValueError("Directory transfers reject absolute paths: " + member.name)
+    if not (member.isfile() or member.isdir() or member.issym() or member.islnk()):
+        raise ValueError("Directory transfers reject special files: " + member.name)
+    return tarfile.data_filter(member, destination)
+
+
+def pack_directory(source, stream):
+    source = Path(source).resolve(strict=True)
+    if not source.is_dir():
+        raise ValueError("Recursive transfer source must be a directory")
+
+    with tarfile.open(fileobj=stream, mode="w|") as archive:
+
+        def add(parent, entry, name):
+            # O_PATH pins the entry without opening a device or following a link.
+            # All metadata, reads and descent then use that held object.
+            descriptor = os.open(entry, os.O_PATH | os.O_NOFOLLOW, dir_fd=parent)
+            try:
+                metadata = os.fstat(descriptor)
+                if stat.S_ISLNK(metadata.st_mode):
+                    member = tarfile.TarInfo(name)
+                    member.type = tarfile.SYMTYPE
+                    member.linkname = os.readlink("", dir_fd=descriptor)
+                    safe_member(member, source)
+                    archive.addfile(member)
+                elif stat.S_ISREG(metadata.st_mode) or stat.S_ISDIR(metadata.st_mode):
+                    opened = SimpleNamespace(name=name, fileno=lambda: descriptor)
+                    member = archive.gettarinfo(arcname=name, fileobj=opened)
+                    safe_member(member, source)
+                    if member.isfile():
+                        # Reopening the pinned descriptor cannot follow a replacement
+                        # pathname. Linux /proc is part of the guest/controller contract.
+                        with open(f"/proc/self/fd/{descriptor}", "rb") as content:
+                            archive.addfile(member, content)
+                    else:
+                        archive.addfile(member)
+                    if member.isdir():
+                        directory = os.open(".", os.O_RDONLY | os.O_DIRECTORY, dir_fd=descriptor)
+                        try:
+                            for child in sorted(os.listdir(directory)):
+                                add(directory, child, name + "/" + child)
+                        finally:
+                            os.close(directory)
+                else:
+                    raise ValueError("Directory transfers reject special files: " + name)
+            finally:
+                os.close(descriptor)
+
+        add(None, source, ".")
+
+
+class TransferReader:
+    """Count physical bytes, including tar's read-ahead and trailing padding."""
+
+    def __init__(self, stream):
+        self.stream = stream
+        self.bytes_read = 0
+
+    def read(self, size=-1):
+        chunk = self.stream.read(size)
+        self.bytes_read += len(chunk)
+        return chunk
+
+
+def unpack_directory(stream, destination, expected_size):
+    # Resolve the parent, never the final component: existing destination symlinks
+    # must be rejected, not followed to a different publication path.
+    destination = Path(destination).absolute()
+    destination = destination.parent.resolve(strict=True) / destination.name
+    if os.path.lexists(destination):
+        raise FileExistsError("Recursive destination already exists: " + str(destination))
+    staged = Path(tempfile.mkdtemp(dir=destination.parent, prefix=".raft-tree-"))
+    try:
+        reader = TransferReader(stream)
+        with tarfile.open(fileobj=reader, mode="r|") as archive:
+            archive.extractall(staged, filter=safe_member)
+        # Tar accepts EOF between entries without a proper end marker. Drain the
+        # stream and check the sender's size before making any tree visible.
+        while reader.read(1024 * 1024):
+            pass
+        if reader.bytes_read != expected_size:
+            raise ValueError("Directory archive size mismatch; transfer did not complete")
+        publish(staged, destination)
+    finally:
+        if staged.exists():
+            shutil.rmtree(staged)
+
+
+def main():
+    cli = argparse.ArgumentParser(description=__doc__)
+    cli.add_argument("operation", choices=["pack", "unpack"])
+    cli.add_argument("path")
+    cli.add_argument("--size", type=int)
+    args = cli.parse_args()
+    if args.operation == "pack":
+        pack_directory(args.path, sys.stdout.buffer)
+    else:
+        if args.size is None or args.size < 0:
+            cli.error("Unpacking requires --size with the sender's archive byte count")
+        unpack_directory(sys.stdin.buffer, args.path, args.size)
+
+
+if __name__ == "__main__":
+    main()

package/skills/raft-cli/SKILL.md

@@ -0,0 +1,117 @@
+---
+name: raft-cli
+description: Create and use private persistent Linux workspaces on configured Linux hosts with Raft and Incus. Use for self-hosted builds, tests, terminals, Docker, file transfer, snapshots, and private desktop access. Hosted Boat uses its separate boat-cli skill.
+---
+
+# Raft
+
+Raft runs unprivileged native ARM64 or AMD64 Incus system containers. They share the host kernel.
+The current deployment does not implement Incus VMs. Do not describe these boxes as VMs or promise cross-architecture emulation.
+
+Run `raft doctor`, `raft limits` and `raft list` before allocating. Keep the exact
+location-qualified handle returned by `new`. There is no implicit current box.
+
+```sh
+box=$(raft new --location lab --ttl 600)
+raft exec "$box" -- timeout 240 bash -lc 'git clone <url> /workspace/project && cd /workspace/project && <test-command>'
+raft download "$box" /workspace/project/report.json ./report.json
+raft destroy "$box"
+```
+
+The default development image has systemd, a guest Docker daemon, Node/npm/pnpm,
+Bun, Deno, Python/pip/venv/uv, Go, Rust, Java, Kotlin, Scala, Ruby, PHP,
+Erlang/Elixir, .NET, R, C/C++ tools, Git, Chromium and FFmpeg. Coding agents are not included in new images; install
+your preferred agent when needed and authenticate separately. Never copy host
+credentials into a box without authorization. Inspect `/opt/raft/` inside the
+box for recorded package versions. Do not assume every language has every
+version manager or that agents have an authenticated account.
+
+Exec passes literal arguments; use `bash -lc` for shell syntax. A failed command
+retains its exit status. `raft ssh "$box"` opens a root terminal through native
+Incus execution over verified host SSH. It does not expose a guest SSH port.
+
+Defaults are the first configured location, one CPU, 2 GiB RAM and 600 seconds. TTL accepts 60 seconds
+to 30 days. The controller admits at most four saved boxes per host; stopped
+boxes count because their disk still occupies space. Use `--memory 4GiB`
+and `--cpu 2` when needed. Btrfs bounds workspace storage to a 60 GiB pool per host.
+It does not guarantee strict per-box disk quotas with nested subvolumes. Check host disk before large builds or forks. `raft limits --location lab --json`
+reports CPU/RAM recommendations for all supported sizes, active allocations and
+free saved slots. Recommendations reserve host resources and remain advisory;
+only the four-saved-box cap is enforced. Disk growth is not included in the counts.
+
+## Persistence and cleanup
+
+Persistent boxes stop and retain disk at expiry. `new --disposable` deletes a
+box and its snapshots on stop or expiry. `raft stop "$box"` releases the
+container's running processes; `raft resume "$box" --ttl 600` starts it again.
+Stop is not pause and does not keep guest RAM allocated. Installed dependencies,
+files and secrets remain on disk. For a temporary run, completion requires
+successful `destroy`, even after command failure. Stop reusable runners when
+finished. Destroy only boxes owned by the current task unless authorized otherwise.
+
+Host systemd timers enforce deadlines every 15 seconds, even when the controller
+is offline. `raft gc` runs those expiry workers manually and deletes expired disposable
+boxes. `raft prune --older-than 7` previews stopped boxes older than seven days;
+`--yes` deletes them. It skips running boxes and boxes without a recorded stop
+time. Never apply prune outside the task-owned scope without explicit permission. A powered-off host enforces expiry when it returns.
+
+Stop a box before a consistent disk snapshot or fork:
+
+```sh
+raft stop "$box"
+raft snapshot "$box" prepared
+raft snapshots "$box"
+copy=$(raft fork "$box" --ttl 600)
+raft restore "$box" prepared
+```
+
+Restore replaces disk contents with the snapshot and retains the destination's
+current instance configuration, including its MAC and resource limits. Use it
+only when the user requested that rollback. A fork copies filesystem data, including any
+credentials installed there. Forks stay on the same host. `new --from "$box/prepared"` starts from a named
+snapshot; `snapshot-delete "$box" prepared` removes it. Resume and fork accept
+optional `--cpu` and `--memory` overrides. Omitted values retain the source limits.
+Forks default to persistent; use `--disposable` explicitly for temporary clones.
+
+`upload/download --recursive` copy directory trees into new destinations without
+merging or overwriting. They require a running guest with Python 3.11.8+ and
+temporary controller disk space. Escaping links and special files are rejected.
+
+## Long commands and previews
+
+`raft exec "$box" --detach -- <command>` returns a job ID. Use `raft status
+"$box" <job>` and `raft logs "$box" <job>` to inspect its systemd state, exit
+status and output. `raft cancel "$box" <job>` stops the job and its child processes. Stop/destroy
+terminates running jobs; this is not automatic checkpointing of processes.
+`raft extend "$box" --ttl 1800` resets a running box's deadline without
+restarting its work. Use resume for stopped boxes. `raft usage "$box"` reports
+cgroup memory/CPU and shared-pool disk space, not a per-box disk quota.
+
+`raft forward "$box" --remote 8080 --local 8080` opens a foreground private
+tunnel. `raft desktop "$box" --local 6080` starts the desktop on demand and
+opens its private tunnel. Use the printed localhost address, with `vnc.html`
+for the viewer. Ctrl+C closes the tunnel. The desktop service stays inside the
+box until the box stops or its service is explicitly stopped.
+
+Public hosting, reverse tunnels, browser-only confinement, agent conversation
+orchestration and KVM-backed VMs are not implemented. Agent binaries can be run
+through exec after authorized sign-in. Do not claim `boat prompt` parity.
+
+## Portable backups
+
+Stop before exporting a private archive, including snapshots:
+
+```sh
+raft stop "$box"
+raft backup "$box" ./workspace.tar.gz
+recovered=$(raft recover ./workspace.tar.gz --location second-host)
+raft resume "$recovered" --ttl 600
+```
+
+Recovery creates a fresh stopped handle and clears the inherited deadline. It
+never overwrites another box. Only import trusted Raft exports; archives retain
+native configuration and guest secrets. Store them privately and check free
+space on the controller and destination host. Large backups hold the host
+lifecycle lock and delay expiry checks. A controller crash can leave an owned
+staging archive or imported box; inspect the reported recovery handle before
+cleanup. This is manual recovery, not a scheduled backup service.