@microck/raft 0.6.0 → 0.7.0

package/README.md

@@ -16,6 +16,12 @@
 control, with command execution, Docker, file transfer, snapshots, forks and a
 private desktop. Raft is not affiliated with Boat.
 
+
+
+https://github.com/user-attachments/assets/ad6b0e67-1e33-4e36-a51b-401f29901eae
+
+
+
 [setup](docs/operations.mdx) | [feature parity](docs/boat-capabilities.md) | [verification](docs/verification.mdx) | [agent skill](skills/raft-cli/SKILL.md) | [docs site](https://raft.micr.dev/)
 
 ## quickstart
@@ -55,7 +61,7 @@ once your host and image are configured, run the commands below. replace `lab`
 with a location from your configuration:
 
 ```sh
-box=$(raft new --location lab --ttl 600)
+box=$(raft new --location lab --ttl 600 --persistent --purpose "Explore Raft" --owner "local operator" --next "Destroy after exploring")
 raft exec "$box" -- node --version
 raft ssh "$box"
 raft stop "$box"
@@ -69,8 +75,9 @@ stopped for 48 hours, and logs each deletion in its journal. disposable
 creation, prune and retention require the current host worker; redeploy from
 the current checkout before using them.
 
-size boxes for the job: `--cpu`, `--memory` and `--disk` take any whole number
-up to the host. on hosts deployed with `--disk-pool SIZE`, `--disk 20GiB` gives a
+size boxes for the job: `--cpu` takes a positive whole CPU count and
+`--memory` takes whole GiB, each up to the host total. `--disk` takes whole
+GiB bounded by the own-disk pool reservation, or `shared`. on hosts deployed with `--disk-pool SIZE`, `--disk 20GiB` gives a
 box its own disk that it, and Docker inside it, cannot grow past; forks and
 templates of it are copy-on-write. raft does not cap how many boxes you keep.
 `raft capacity` prints an advisory estimate of what fits; it never blocks a box.
@@ -102,7 +109,8 @@ and [Docker](docs/guides/docker.mdx).
 
 See [project setup and templates](docs/guides/project-setup.mdx),
 [persistent services](docs/guides/long-running-services.mdx),
-[scripting and CI](docs/guides/scripting-ci.mdx), and
+[scripting and CI](docs/guides/scripting-ci.mdx),
+[disposable CI jobs](docs/guides/ci-jobs.mdx), and
 [upgrades](docs/guides/upgrades.mdx) for complete operator workflows.
 
 stop the source box before taking a snapshot, restoring one or making a fork.
@@ -150,6 +158,7 @@ records differences, sources and test coverage.
 | CPU/memory sizing and usage | ✅ | ✅ | Any whole CPU count and GiB size up to the host. Usage covers the shared pool or the box's own disk |
 | Host capacity estimates | ❌ | ✅ | Advisory only; Raft has no box count limit. Boat reports plan limits |
 | Root terminal and command execution | ✅ | ✅ | Host SSH transport |
+| Disposable foreground CI jobs | ✅ | Preview | Unreleased 0.7.0 `raft run`: fixed timeout, Git-aware upload, artifact collection and guarded cleanup; absent from published 0.6.0 |
 | Background jobs, logs and cancellation | ✅ | ✅ | No process checkpoints |
 | File upload/download | ✅ | ✅ | Files and recursive trees; new directory destinations only |
 | Snapshots and same-host forks | ✅ | ✅ | Stopped source, checked under the lifecycle lock |

package/deploy/build-image.py

@@ -41,15 +41,17 @@ def build(host, alias):
         host,
         prefix + " launch " + fingerprint + " raft-builder -c limits.cpu=2 -c limits.memory=4GiB",
     )
+    # Stage outside /tmp: its boot-time tmpfs mount can hide an early file push.
     for source, guest, mode in [
-        ("images/incus/build.sh", "/tmp/raft-build.sh", "755"),
+        ("images/incus/build.sh", "/root/raft-build.sh", "755"),
         ("images/incus/desktop.sh", "/usr/local/bin/raft-desktop", "755"),
         ("images/incus/raft-desktop.service", "/etc/systemd/system/raft-desktop.service", "644"),
     ]:
         staged = "/tmp/" + Path(guest).name
         copy(host, ROOT / source, staged)
         remote(host, prefix + f" file push {staged} raft-builder{guest} --mode {mode}")
-    remote(host, prefix + " exec raft-builder -- bash /tmp/raft-build.sh")
+    remote(host, prefix + " exec raft-builder -- bash /root/raft-build.sh")
+    remote(host, prefix + " exec raft-builder -- rm /root/raft-build.sh")
     remote(host, prefix + " stop raft-builder --timeout 30")
     print(
         remote(

package/deploy/verify-controls.py

@@ -88,19 +88,19 @@ def verify(source, target, directory, development_image=False):
                 raise RuntimeError("Detached job did not start")
             time.sleep(0.2)
         child = live.execute(box, "cat", "/workspace/child-pid")
-        assert "ActiveState=active" in live.raft("status", box, job).stdout
+        assert json.loads(live.raft("status", box, job).stdout)["state"] == "running"
         live.raft("extend", box, "--ttl", "1200")
         extended = json.loads(live.raft("info", box).stdout)
         assert (
             int(extended["config"]["user.raft.expires"])
             > int(live.execute(box, "date", "+%s")) + 1100
         )
-        assert "ActiveState=active" in live.raft("status", box, job).stdout
-        # Exercise journal retention without changing any host service or other box.
+        assert json.loads(live.raft("status", box, job).stdout)["state"] == "running"
+        # Guest journal rotation cannot erase host-retained job state.
         live.execute(box, "journalctl", "--rotate", "--vacuum-size=1")
-        assert "ActiveState=active" in live.raft("status", box, job).stdout
+        assert json.loads(live.raft("status", box, job).stdout)["state"] == "running"
         live.raft("cancel", box, job)
-        assert "ActiveState=inactive" in live.raft("status", box, job).stdout
+        assert live.wait_job(box, job)["state"] == "cancelled"
         assert live.raft("logs", box, job).returncode == 0
         assert (
             live.raft("exec", box, "--", "test", "-e", "/proc/" + child, check=False).returncode

package/deploy/verify-features.py

@@ -32,6 +32,8 @@ from raft import (  # noqa: E402
     RETENTION_SINCE_KEY,
     disk_origin,
     host_problems,
+    host_inventory,
+    host_policy,
     incus,
     inventory,
     parse_handle,
@@ -694,8 +696,12 @@ def verify_readiness(location, expect_stale=False):
         (home / ".ssh").symlink_to(Path.home() / ".ssh", target_is_directory=True)
         config = home / ".config/raft/incus.json"
         config.parent.mkdir(parents=True)
-        config.write_text(json.dumps({location: settings(location)}))
-        response = live.raft("doctor", check=False, env={**os.environ, "HOME": str(home)})
+        config.write_text(
+            json.dumps({"offline": {"ssh": "raft-offline.invalid"}, location: settings(location)})
+        )
+        response = live.raft(
+            "doctor", "--location", location, check=False, env={**os.environ, "HOME": str(home)}
+        )
     assert location + ": " in response.stdout, response
     if expect_stale:
         assert response.returncode == 1, response
@@ -709,6 +715,32 @@ def verify_readiness(location, expect_stale=False):
     print(f"{location}: doctor {'rejected stale worker' if expect_stale else 'passed'}", flush=True)
 
 
+def verify_read_reports(location):
+    """Compare batched reads with independent native reads on a quiescent host."""
+    policy, boxes = host_inventory(location)
+    assert policy == host_policy(location)
+    assert sorted(boxes, key=lambda box: box["name"]) == sorted(
+        inventory(location), key=lambda box: box["name"]
+    )
+    with tempfile.TemporaryDirectory(prefix="raft-config-test-") as work:
+        home = Path(work)
+        (home / ".ssh").symlink_to(Path.home() / ".ssh", target_is_directory=True)
+        config = home / ".config/raft/incus.json"
+        config.parent.mkdir(parents=True)
+        config.write_text(
+            json.dumps({"offline": {"ssh": "raft-offline.invalid"}, location: settings(location)})
+        )
+        env = {**os.environ, "HOME": str(home), "RAFT_NO_UPDATE_CHECK": "1"}
+        report = live.raft("config", "--location", location, env=env)
+        assert json.loads(report.stdout)["policy"] == {location: policy}
+        rejected = live.raft(
+            "config", "--location", location, "set", "cpu", "2", env=env, check=False
+        )
+        assert rejected.returncode == 1 and "controller-wide" in rejected.stderr
+        assert not (config.parent / "config.json").exists()
+    print(f"{location}: batched metadata and offline-host isolation passed", flush=True)
+
+
 def verify_transaction_stdin(location):
     program = """import sys, tempfile
 from raft import transaction
@@ -1276,4 +1308,5 @@ if __name__ == "__main__":
     verify_host_preflight()
     verify_archives()
     if args.location:
+        verify_read_reports(args.location)
         verify(args.location, args.isolated_worker)

package/deploy/verify-jobs.py

@@ -0,0 +1,264 @@
+"""Real process/store checks and optional disposable-host retained-job verification."""
+
+import argparse
+import importlib.util
+import json
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import time
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+import raft_jobs as jobs  # noqa: E402
+
+
+def stored(job, *, state="completed", finished=None):
+    return {
+        "job": job,
+        "box": "rf-" + "a" * 16,
+        "state": state,
+        "exitCode": 7,
+        "finishedAt": int(time.time()) if finished is None else finished,
+        "bootId": "old-boot",
+    }
+
+
+def verify():
+    with tempfile.TemporaryDirectory(prefix="raft-jobs-") as directory:
+        root = Path(directory)
+        with jobs.private_store(root):
+            pass
+        status, dropped = jobs.capture(
+            root,
+            [
+                sys.executable,
+                "-c",
+                "import os,sys; os.write(1,b'x'*(2*1024*1024)); os.write(2,b'err'); sys.exit(7)",
+            ],
+        )
+        assert status == 7 and dropped == jobs.LOG_LIMIT + 3
+        assert (root / "output").read_bytes() == b"x" * jobs.LOG_LIMIT
+        (root / "output").unlink()
+        # A real file-size limit fails the first record write before any service launch.
+        rejected = subprocess.run(
+            [
+                sys.executable,
+                "-c",
+                "import resource,signal,sys; from pathlib import Path; import raft_jobs; "
+                "signal.signal(signal.SIGXFSZ, signal.SIG_IGN); "
+                "resource.setrlimit(resource.RLIMIT_FSIZE, (0, 0)); "
+                "raft_jobs.start(Path(sys.argv[1]), 'rf-'+'a'*16, 'rfcmd-'+'d'*32, ['true'], '')",
+                str(root),
+            ],
+            cwd=ROOT,
+            capture_output=True,
+            text=True,
+        )
+        assert rejected.returncode != 0 and "File too large" in rejected.stderr
+        assert not list(root.iterdir()), "Failed allocation consumed a record slot"
+        old = root / ("rfcmd-" + "a" * 32)
+        old.mkdir()
+        jobs.save(old, stored(old.name, finished=int(time.time()) - jobs.RETENTION - 1))
+        current = root / ("rfcmd-" + "b" * 32)
+        current.mkdir()
+        jobs.save(current, stored(current.name))
+        interrupted = root / ("rfcmd-" + "c" * 32)
+        interrupted.mkdir()
+        jobs.save(interrupted, stored(interrupted.name, state="running"))
+        incomplete = root / ("rfcmd-" + "e" * 32)
+        incomplete.mkdir()
+        damaged = root / ("rfcmd-" + "f" * 32)
+        damaged.mkdir()
+        (damaged / "result.json").write_text('{"state":')
+        (damaged / "output").write_bytes(b"retained evidence")
+        jobs.prune(root)
+        assert incomplete.exists() and (damaged / "output").read_bytes() == b"retained evidence"
+        for unreadable in (incomplete, damaged):
+            try:
+                jobs.load(unreadable)
+            except ValueError as error:
+                assert str(unreadable / "result.json") in str(error)
+            else:
+                raise AssertionError("Unreadable record accepted")
+        inspected = subprocess.run(
+            [
+                sys.executable,
+                "-c",
+                "from pathlib import Path; import sys,raft_jobs; "
+                "root=Path(sys.argv.pop(1)); raft_jobs.main(root)",
+                str(root),
+                "status",
+                "rf-" + "a" * 16,
+                current.name,
+            ],
+            cwd=ROOT,
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        assert json.loads(inspected.stdout)["exitCode"] == 7
+        for malformed in (None, {}, {**stored(damaged.name), "finishedAt": "invalid"}):
+            (damaged / "result.json").write_text(json.dumps(malformed))
+            jobs.prune(root)
+            assert jobs.load(current)["exitCode"] == 7
+            try:
+                jobs.load(damaged)
+            except ValueError:
+                pass
+            else:
+                raise AssertionError("Malformed record accepted")
+        assert not old.exists() and current.exists()
+        assert jobs.load(interrupted)["state"] == "interrupted"
+        assert jobs.load(interrupted)["exitCode"] is None
+        root.chmod(0o755)
+        try:
+            with jobs.private_store(root):
+                raise AssertionError("World-readable job store accepted")
+        except ValueError:
+            pass
+        finally:
+            root.chmod(0o700)
+        for index in range(jobs.RECORD_LIMIT - len(list(root.iterdir()))):
+            (root / f"slot-{index}").mkdir()
+        try:
+            jobs.start(root, "rf-" + "a" * 16, "rfcmd-" + "d" * 32, ["true"], "")
+        except ValueError as error:
+            assert "full" in str(error)
+        else:
+            raise AssertionError("Full store accepted a new job")
+    print(
+        "Real process capture, damaged-record isolation, retention, reboot interruption and store limits passed"
+    )
+
+
+def native(location):
+    spec = importlib.util.spec_from_file_location("live", ROOT / "deploy/verify-live.py")
+    live = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(live)
+    box = live.raft(
+        "new",
+        "--location",
+        location,
+        "--persistent",
+        "--ttl",
+        "600",
+        "--purpose",
+        "Retained job verification",
+        "--owner",
+        "verify-jobs",
+    ).stdout.strip()
+    destroyed = False
+    try:
+        deadline = time.monotonic() + 40
+        while live.raft(
+            "exec", box, "--", "systemctl", "show", "-p", "Version", check=False
+        ).returncode:
+            assert time.monotonic() < deadline, "Guest systemd did not become ready"
+            time.sleep(0.2)
+        completed = live.raft(
+            "exec", box, "--detach", "--", "sh", "-c", "printf retained-output; exit 23"
+        ).stdout.strip()
+        record = live.wait_job(box, completed)
+        output = live.raft("logs", box, completed).stdout
+        if record["exitCode"] != 23 or record["state"] != "completed":
+            unit = live.raft("exec", box, "--", "systemctl", "show", completed, check=False)
+            raise AssertionError(
+                {"record": record, "output": output, "unit": unit.stdout, "error": unit.stderr}
+            )
+        assert output == "retained-output"
+        for signum in (1, 2, 13, 15):
+            signalled = live.raft(
+                "exec",
+                box,
+                "--detach",
+                "--",
+                "python3",
+                "-c",
+                f"import os,signal; signal.signal({signum},signal.SIG_DFL); "
+                f"os.kill(os.getpid(),{signum})",
+            ).stdout.strip()
+            record = live.wait_job(box, signalled)
+            assert record["state"] == "completed" and record["exitCode"] == 128 + signum, (
+                record,
+                live.raft("logs", box, signalled).stdout,
+            )
+        active = live.raft(
+            "exec", box, "--detach", "--", "sh", "-c", "echo ready; sleep 600"
+        ).stdout.strip()
+        deadline = time.monotonic() + 20
+        while "ready" not in live.raft("logs", box, active).stdout:
+            assert time.monotonic() < deadline
+            time.sleep(0.2)
+        live.raft("cancel", box, active)
+        record = live.wait_job(box, active)
+        assert record["state"] == "cancelled", (record, live.raft("logs", box, active).stdout)
+        orphan = live.raft(
+            "exec", box, "--detach", "--", "sh", "-c", "echo orphan-ready; exec sleep 600"
+        ).stdout.strip()
+        deadline = time.monotonic() + 20
+        while "orphan-ready" not in live.raft("logs", box, orphan).stdout:
+            assert time.monotonic() < deadline, "Orphan test payload did not start"
+            time.sleep(0.2)
+        live.transaction(location, 'systemctl kill --signal=SIGKILL "$1"', "raft-" + orphan)
+        record = live.wait_job(box, orphan)
+        assert record["state"] == "interrupted", record
+        assert int(live.execute(box, "systemctl", "show", orphan, "-p", "MainPID", "--value")) > 0
+        live.raft("cancel", box, orphan)
+        record = json.loads(live.raft("status", box, orphan).stdout)
+        assert record["state"] == "cancelled" and record["exitCode"] is None, record
+        assert record["finishedAt"] is not None
+        # ExecMainPID retains the historical PID after exit; MainPID tracks the live process.
+        assert int(live.execute(box, "systemctl", "show", orphan, "-p", "MainPID", "--value")) == 0
+        live.raft("cancel", box, orphan)
+        assert json.loads(live.raft("status", box, orphan).stdout) == record
+        interrupted = live.raft("exec", box, "--detach", "--", "sleep", "600").stdout.strip()
+        zero_on_stop = live.raft(
+            "exec",
+            box,
+            "--detach",
+            "--",
+            "python3",
+            "-u",
+            "-c",
+            "import signal,sys,time; signal.signal(signal.SIGTERM, lambda *_: sys.exit(0)); "
+            "print('shutdown-handler-ready'); time.sleep(600)",
+        ).stdout.strip()
+        deadline = time.monotonic() + 20
+        while "shutdown-handler-ready" not in live.raft("logs", box, zero_on_stop).stdout:
+            assert time.monotonic() < deadline, "Shutdown handler did not start"
+            time.sleep(0.2)
+        live.raft("stop", box)
+        for stopped_job in (interrupted, zero_on_stop):
+            record = live.wait_job(box, stopped_job)
+            assert record["state"] == "interrupted" and record["exitCode"] is None, (
+                record,
+                live.raft("logs", box, stopped_job).stdout,
+            )
+        assert live.raft("logs", box, completed).stdout == "retained-output"
+        count_records = "find /var/lib/raft-jobs -mindepth 1 -maxdepth 1 -type d | wc -l"
+        before = live.transaction(location, count_records)
+        refused = live.raft("exec", box, "--detach", "--", "true", check=False)
+        assert refused.returncode == 1 and not refused.stdout, refused
+        assert "no job was created" in refused.stderr, refused.stderr
+        assert live.transaction(location, count_records) == before
+        live.raft("destroy", box)
+        destroyed = True
+        assert json.loads(live.raft("status", box, completed).stdout)["exitCode"] == 23
+        assert live.raft("logs", box, completed).stdout == "retained-output"
+        wrong = location + ":rf-" + "0" * 16
+        assert live.raft("logs", wrong, completed, check=False).returncode == 1
+        print(f"{location}: completed, cancelled and interrupted results survive stop and deletion")
+    finally:
+        if not destroyed:
+            live.raft("destroy", box)
+
+
+if __name__ == "__main__":
+    cli = argparse.ArgumentParser(description=__doc__)
+    cli.add_argument("--location")
+    args = cli.parse_args()
+    verify()
+    if args.location:
+        native(args.location)

package/deploy/verify-limits.py

@@ -299,8 +299,26 @@ def disk_checks(location):
             time.sleep(2)
         assert disk_size(box)[0] == total, disk_size(box)
         # Removing the container frees the space; restart Docker in case it lost state.
-        live.raft("exec", box, "--", "docker", "rm", "-f", "raft-fill", check=False)
-        live.execute(box, "systemctl", "restart", "docker")
+        removed = live.raft("exec", box, "--", "docker", "rm", "-f", "raft-fill", check=False)
+        restarted = live.raft("exec", box, "--", "systemctl", "restart", "docker", check=False)
+        if restarted.returncode:
+            print(
+                "Docker removal after disk exhaustion:", removed.stdout, removed.stderr, flush=True
+            )
+            journal = live.raft(
+                "exec",
+                box,
+                "--",
+                "journalctl",
+                "--no-pager",
+                "-n",
+                "100",
+                "-u",
+                "docker",
+                check=False,
+            )
+            print(journal.stdout, journal.stderr, flush=True)
+            raise RuntimeError("Docker restart after disk exhaustion failed: " + restarted.stderr)
         live.raft("exec", box, "--", "docker", "rm", "-f", "raft-fill", check=False)
         assert disk_size(box)[1] > GIB, disk_size(box)
         assert own_pool(location)["reservedBytes"] == created["reservedBytes"]

package/deploy/verify-live.py

@@ -57,6 +57,16 @@ def wait_running(box):
     raise RuntimeError("Guest Docker and SSH did not become ready")
 
 
+def wait_job(box, job):
+    deadline = time.monotonic() + 30
+    while time.monotonic() < deadline:
+        record = json.loads(raft("status", box, job).stdout)
+        if record["state"] in {"completed", "cancelled", "interrupted", "failed"}:
+            return record
+        time.sleep(0.2)
+    raise RuntimeError("Detached job did not finish: " + job)
+
+
 def terminal_session(box):
     master, slave = pty.openpty()
     process = subprocess.Popen([*CLI, "ssh", box], stdin=slave, stdout=slave, stderr=slave)
@@ -796,8 +806,7 @@ def verify(location, extended=False, restart_incus=False, reboot_host=False):
         job = raft(
             "exec", box, "--detach", "--", "bash", "-lc", "echo detached-ok; exit 23"
         ).stdout.strip()
-        time.sleep(2)
-        assert "ExecMainStatus=23" in raft("status", box, job).stdout
+        assert wait_job(box, job)["exitCode"] == 23
         assert "detached-ok" in raft("logs", box, job).stdout
         execute(box, "bash", "-lc", "echo saved > /workspace/persistent")
         raft("stop", box)
@@ -824,7 +833,7 @@ def verify(location, extended=False, restart_incus=False, reboot_host=False):
             fetch_through_tunnel(["forward", box, "--remote", "8080"], "/persistent").strip()
             == b"saved"
         )
-        assert "ActiveState=active" in raft("status", box, server).stdout
+        assert json.loads(raft("status", box, server).stdout)["state"] == "running"
         assert b"noVNC" in fetch_through_tunnel(["desktop", box], "/vnc.html", timeout=20)
         assert b"noVNC" in fetch_through_tunnel(
             ["desktop", box], "/vnc.html", timeout=20, default_port=6080