@microck/raft 0.5.0 → 0.7.0

package/deploy/verify-jobs.py

@@ -0,0 +1,264 @@
+"""Real process/store checks and optional disposable-host retained-job verification."""
+
+import argparse
+import importlib.util
+import json
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import time
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+import raft_jobs as jobs  # noqa: E402
+
+
+def stored(job, *, state="completed", finished=None):
+    return {
+        "job": job,
+        "box": "rf-" + "a" * 16,
+        "state": state,
+        "exitCode": 7,
+        "finishedAt": int(time.time()) if finished is None else finished,
+        "bootId": "old-boot",
+    }
+
+
+def verify():
+    with tempfile.TemporaryDirectory(prefix="raft-jobs-") as directory:
+        root = Path(directory)
+        with jobs.private_store(root):
+            pass
+        status, dropped = jobs.capture(
+            root,
+            [
+                sys.executable,
+                "-c",
+                "import os,sys; os.write(1,b'x'*(2*1024*1024)); os.write(2,b'err'); sys.exit(7)",
+            ],
+        )
+        assert status == 7 and dropped == jobs.LOG_LIMIT + 3
+        assert (root / "output").read_bytes() == b"x" * jobs.LOG_LIMIT
+        (root / "output").unlink()
+        # A real file-size limit fails the first record write before any service launch.
+        rejected = subprocess.run(
+            [
+                sys.executable,
+                "-c",
+                "import resource,signal,sys; from pathlib import Path; import raft_jobs; "
+                "signal.signal(signal.SIGXFSZ, signal.SIG_IGN); "
+                "resource.setrlimit(resource.RLIMIT_FSIZE, (0, 0)); "
+                "raft_jobs.start(Path(sys.argv[1]), 'rf-'+'a'*16, 'rfcmd-'+'d'*32, ['true'], '')",
+                str(root),
+            ],
+            cwd=ROOT,
+            capture_output=True,
+            text=True,
+        )
+        assert rejected.returncode != 0 and "File too large" in rejected.stderr
+        assert not list(root.iterdir()), "Failed allocation consumed a record slot"
+        old = root / ("rfcmd-" + "a" * 32)
+        old.mkdir()
+        jobs.save(old, stored(old.name, finished=int(time.time()) - jobs.RETENTION - 1))
+        current = root / ("rfcmd-" + "b" * 32)
+        current.mkdir()
+        jobs.save(current, stored(current.name))
+        interrupted = root / ("rfcmd-" + "c" * 32)
+        interrupted.mkdir()
+        jobs.save(interrupted, stored(interrupted.name, state="running"))
+        incomplete = root / ("rfcmd-" + "e" * 32)
+        incomplete.mkdir()
+        damaged = root / ("rfcmd-" + "f" * 32)
+        damaged.mkdir()
+        (damaged / "result.json").write_text('{"state":')
+        (damaged / "output").write_bytes(b"retained evidence")
+        jobs.prune(root)
+        assert incomplete.exists() and (damaged / "output").read_bytes() == b"retained evidence"
+        for unreadable in (incomplete, damaged):
+            try:
+                jobs.load(unreadable)
+            except ValueError as error:
+                assert str(unreadable / "result.json") in str(error)
+            else:
+                raise AssertionError("Unreadable record accepted")
+        inspected = subprocess.run(
+            [
+                sys.executable,
+                "-c",
+                "from pathlib import Path; import sys,raft_jobs; "
+                "root=Path(sys.argv.pop(1)); raft_jobs.main(root)",
+                str(root),
+                "status",
+                "rf-" + "a" * 16,
+                current.name,
+            ],
+            cwd=ROOT,
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        assert json.loads(inspected.stdout)["exitCode"] == 7
+        for malformed in (None, {}, {**stored(damaged.name), "finishedAt": "invalid"}):
+            (damaged / "result.json").write_text(json.dumps(malformed))
+            jobs.prune(root)
+            assert jobs.load(current)["exitCode"] == 7
+            try:
+                jobs.load(damaged)
+            except ValueError:
+                pass
+            else:
+                raise AssertionError("Malformed record accepted")
+        assert not old.exists() and current.exists()
+        assert jobs.load(interrupted)["state"] == "interrupted"
+        assert jobs.load(interrupted)["exitCode"] is None
+        root.chmod(0o755)
+        try:
+            with jobs.private_store(root):
+                raise AssertionError("World-readable job store accepted")
+        except ValueError:
+            pass
+        finally:
+            root.chmod(0o700)
+        for index in range(jobs.RECORD_LIMIT - len(list(root.iterdir()))):
+            (root / f"slot-{index}").mkdir()
+        try:
+            jobs.start(root, "rf-" + "a" * 16, "rfcmd-" + "d" * 32, ["true"], "")
+        except ValueError as error:
+            assert "full" in str(error)
+        else:
+            raise AssertionError("Full store accepted a new job")
+    print(
+        "Real process capture, damaged-record isolation, retention, reboot interruption and store limits passed"
+    )
+
+
+def native(location):
+    spec = importlib.util.spec_from_file_location("live", ROOT / "deploy/verify-live.py")
+    live = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(live)
+    box = live.raft(
+        "new",
+        "--location",
+        location,
+        "--persistent",
+        "--ttl",
+        "600",
+        "--purpose",
+        "Retained job verification",
+        "--owner",
+        "verify-jobs",
+    ).stdout.strip()
+    destroyed = False
+    try:
+        deadline = time.monotonic() + 40
+        while live.raft(
+            "exec", box, "--", "systemctl", "show", "-p", "Version", check=False
+        ).returncode:
+            assert time.monotonic() < deadline, "Guest systemd did not become ready"
+            time.sleep(0.2)
+        completed = live.raft(
+            "exec", box, "--detach", "--", "sh", "-c", "printf retained-output; exit 23"
+        ).stdout.strip()
+        record = live.wait_job(box, completed)
+        output = live.raft("logs", box, completed).stdout
+        if record["exitCode"] != 23 or record["state"] != "completed":
+            unit = live.raft("exec", box, "--", "systemctl", "show", completed, check=False)
+            raise AssertionError(
+                {"record": record, "output": output, "unit": unit.stdout, "error": unit.stderr}
+            )
+        assert output == "retained-output"
+        for signum in (1, 2, 13, 15):
+            signalled = live.raft(
+                "exec",
+                box,
+                "--detach",
+                "--",
+                "python3",
+                "-c",
+                f"import os,signal; signal.signal({signum},signal.SIG_DFL); "
+                f"os.kill(os.getpid(),{signum})",
+            ).stdout.strip()
+            record = live.wait_job(box, signalled)
+            assert record["state"] == "completed" and record["exitCode"] == 128 + signum, (
+                record,
+                live.raft("logs", box, signalled).stdout,
+            )
+        active = live.raft(
+            "exec", box, "--detach", "--", "sh", "-c", "echo ready; sleep 600"
+        ).stdout.strip()
+        deadline = time.monotonic() + 20
+        while "ready" not in live.raft("logs", box, active).stdout:
+            assert time.monotonic() < deadline
+            time.sleep(0.2)
+        live.raft("cancel", box, active)
+        record = live.wait_job(box, active)
+        assert record["state"] == "cancelled", (record, live.raft("logs", box, active).stdout)
+        orphan = live.raft(
+            "exec", box, "--detach", "--", "sh", "-c", "echo orphan-ready; exec sleep 600"
+        ).stdout.strip()
+        deadline = time.monotonic() + 20
+        while "orphan-ready" not in live.raft("logs", box, orphan).stdout:
+            assert time.monotonic() < deadline, "Orphan test payload did not start"
+            time.sleep(0.2)
+        live.transaction(location, 'systemctl kill --signal=SIGKILL "$1"', "raft-" + orphan)
+        record = live.wait_job(box, orphan)
+        assert record["state"] == "interrupted", record
+        assert int(live.execute(box, "systemctl", "show", orphan, "-p", "MainPID", "--value")) > 0
+        live.raft("cancel", box, orphan)
+        record = json.loads(live.raft("status", box, orphan).stdout)
+        assert record["state"] == "cancelled" and record["exitCode"] is None, record
+        assert record["finishedAt"] is not None
+        # ExecMainPID retains the historical PID after exit; MainPID tracks the live process.
+        assert int(live.execute(box, "systemctl", "show", orphan, "-p", "MainPID", "--value")) == 0
+        live.raft("cancel", box, orphan)
+        assert json.loads(live.raft("status", box, orphan).stdout) == record
+        interrupted = live.raft("exec", box, "--detach", "--", "sleep", "600").stdout.strip()
+        zero_on_stop = live.raft(
+            "exec",
+            box,
+            "--detach",
+            "--",
+            "python3",
+            "-u",
+            "-c",
+            "import signal,sys,time; signal.signal(signal.SIGTERM, lambda *_: sys.exit(0)); "
+            "print('shutdown-handler-ready'); time.sleep(600)",
+        ).stdout.strip()
+        deadline = time.monotonic() + 20
+        while "shutdown-handler-ready" not in live.raft("logs", box, zero_on_stop).stdout:
+            assert time.monotonic() < deadline, "Shutdown handler did not start"
+            time.sleep(0.2)
+        live.raft("stop", box)
+        for stopped_job in (interrupted, zero_on_stop):
+            record = live.wait_job(box, stopped_job)
+            assert record["state"] == "interrupted" and record["exitCode"] is None, (
+                record,
+                live.raft("logs", box, stopped_job).stdout,
+            )
+        assert live.raft("logs", box, completed).stdout == "retained-output"
+        count_records = "find /var/lib/raft-jobs -mindepth 1 -maxdepth 1 -type d | wc -l"
+        before = live.transaction(location, count_records)
+        refused = live.raft("exec", box, "--detach", "--", "true", check=False)
+        assert refused.returncode == 1 and not refused.stdout, refused
+        assert "no job was created" in refused.stderr, refused.stderr
+        assert live.transaction(location, count_records) == before
+        live.raft("destroy", box)
+        destroyed = True
+        assert json.loads(live.raft("status", box, completed).stdout)["exitCode"] == 23
+        assert live.raft("logs", box, completed).stdout == "retained-output"
+        wrong = location + ":rf-" + "0" * 16
+        assert live.raft("logs", wrong, completed, check=False).returncode == 1
+        print(f"{location}: completed, cancelled and interrupted results survive stop and deletion")
+    finally:
+        if not destroyed:
+            live.raft("destroy", box)
+
+
+if __name__ == "__main__":
+    cli = argparse.ArgumentParser(description=__doc__)
+    cli.add_argument("--location")
+    args = cli.parse_args()
+    verify()
+    if args.location:
+        native(args.location)

package/deploy/verify-limits.py

@@ -108,7 +108,8 @@ def policy_checks():
 
 
 def command(*args):
-    return subprocess.check_output([*CLI, *args], text=True).strip()
+    # The private controller HOME keeps this controller's defaults for new out.
+    return subprocess.check_output([*CLI, *args], text=True, env=live.CONTROLLER_ENV).strip()
 
 
 def report(location):
@@ -298,8 +299,26 @@ def disk_checks(location):
             time.sleep(2)
         assert disk_size(box)[0] == total, disk_size(box)
         # Removing the container frees the space; restart Docker in case it lost state.
-        live.raft("exec", box, "--", "docker", "rm", "-f", "raft-fill", check=False)
-        live.execute(box, "systemctl", "restart", "docker")
+        removed = live.raft("exec", box, "--", "docker", "rm", "-f", "raft-fill", check=False)
+        restarted = live.raft("exec", box, "--", "systemctl", "restart", "docker", check=False)
+        if restarted.returncode:
+            print(
+                "Docker removal after disk exhaustion:", removed.stdout, removed.stderr, flush=True
+            )
+            journal = live.raft(
+                "exec",
+                box,
+                "--",
+                "journalctl",
+                "--no-pager",
+                "-n",
+                "100",
+                "-u",
+                "docker",
+                check=False,
+            )
+            print(journal.stdout, journal.stderr, flush=True)
+            raise RuntimeError("Docker restart after disk exhaustion failed: " + restarted.stderr)
         live.raft("exec", box, "--", "docker", "rm", "-f", "raft-fill", check=False)
         assert disk_size(box)[1] > GIB, disk_size(box)
         assert own_pool(location)["reservedBytes"] == created["reservedBytes"]

package/deploy/verify-live.py

@@ -7,6 +7,7 @@ import os
 import pty
 import select
 import shlex
+import shutil
 from pathlib import Path
 import socket
 import subprocess
@@ -19,11 +20,24 @@ ROOT = Path(__file__).resolve().parent.parent
 CLI = [sys.executable, str(ROOT / "raft.py")]
 USER_AGENT = "OpenAI File Downloader, XaiImageApiFetch/1.0"
 sys.path.insert(0, str(ROOT))
-from raft import configuration, incus, remote, settings, transaction  # noqa: E402
+from raft import CONFIG, configuration, incus, remote, settings, transaction  # noqa: E402
+
+# Suites run the CLI in a private HOME that holds only the location inventory, so
+# this controller's defaults for new (config.json) never reach them: a disposable
+# default would delete boxes a suite stops and expects to keep, and size defaults
+# would change asserted limits. OpenSSH reads keys and aliases from the passwd
+# home, not HOME. The inventory is copied once; suites that edit it use their own.
+CONTROLLER_HOME = tempfile.TemporaryDirectory(prefix="raft-verify-home-")
+if CONFIG.exists():
+    (Path(CONTROLLER_HOME.name) / ".config/raft").mkdir(parents=True)
+    shutil.copy2(CONFIG, Path(CONTROLLER_HOME.name) / ".config/raft/incus.json")
+CONTROLLER_ENV = {**os.environ, "HOME": CONTROLLER_HOME.name}
 
 
 def raft(*arguments, check=True, env=None):
-    response = subprocess.run([*CLI, *arguments], capture_output=True, text=True, env=env)
+    response = subprocess.run(
+        [*CLI, *arguments], capture_output=True, text=True, env=env or CONTROLLER_ENV
+    )
     if check and response.returncode:
         raise RuntimeError(f"Raft {arguments!r} failed: {response.stderr.strip()}")
     return response
@@ -43,6 +57,16 @@ def wait_running(box):
     raise RuntimeError("Guest Docker and SSH did not become ready")
 
 
+def wait_job(box, job):
+    deadline = time.monotonic() + 30
+    while time.monotonic() < deadline:
+        record = json.loads(raft("status", box, job).stdout)
+        if record["state"] in {"completed", "cancelled", "interrupted", "failed"}:
+            return record
+        time.sleep(0.2)
+    raise RuntimeError("Detached job did not finish: " + job)
+
+
 def terminal_session(box):
     master, slave = pty.openpty()
     process = subprocess.Popen([*CLI, "ssh", box], stdin=slave, stdout=slave, stderr=slave)
@@ -352,7 +376,9 @@ def browser_session(box):
                 assert.equal(response.result.errorText, undefined);
                 socket.send(JSON.stringify(viewerProof));
             } else if (response.id === 4) {
-                const viewer = JSON.parse(response.result.result.value);
+                // While noVNC is still loading, the evaluation can return no value.
+                const value = response.result.result.value;
+                const viewer = typeof value === 'string' ? JSON.parse(value) : {connected: false};
                 if (!viewer.connected) {
                     setTimeout(() => socket.send(JSON.stringify(viewerProof)), 200); return;
                 }
@@ -780,8 +806,7 @@ def verify(location, extended=False, restart_incus=False, reboot_host=False):
         job = raft(
             "exec", box, "--detach", "--", "bash", "-lc", "echo detached-ok; exit 23"
         ).stdout.strip()
-        time.sleep(2)
-        assert "ExecMainStatus=23" in raft("status", box, job).stdout
+        assert wait_job(box, job)["exitCode"] == 23
         assert "detached-ok" in raft("logs", box, job).stdout
         execute(box, "bash", "-lc", "echo saved > /workspace/persistent")
         raft("stop", box)
@@ -808,7 +833,7 @@ def verify(location, extended=False, restart_incus=False, reboot_host=False):
             fetch_through_tunnel(["forward", box, "--remote", "8080"], "/persistent").strip()
             == b"saved"
         )
-        assert "ActiveState=active" in raft("status", box, server).stdout
+        assert json.loads(raft("status", box, server).stdout)["state"] == "running"
         assert b"noVNC" in fetch_through_tunnel(["desktop", box], "/vnc.html", timeout=20)
         assert b"noVNC" in fetch_through_tunnel(
             ["desktop", box], "/vnc.html", timeout=20, default_port=6080

package/deploy/verify-output.py

@@ -0,0 +1,208 @@
+"""Verify JSONL bytes, errors, completion discovery and native command status."""
+
+import argparse
+import base64
+import importlib.util
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+import raft  # noqa: E402
+import raft_output  # noqa: E402
+
+
+def events(command, env):
+    response = subprocess.run(command, env=env, capture_output=True)
+    records = [json.loads(line) for line in response.stdout.splitlines()]
+    assert not response.stderr, response.stderr
+    assert records[-1] == {"type": "exit", "code": response.returncode}, records
+    return response, records
+
+
+def stream(records, kind):
+    return b"".join(base64.b64decode(r["data"]) for r in records if r["type"] == kind)
+
+
+def verify():
+    with tempfile.TemporaryDirectory(prefix="raft-output-") as directory:
+        env = {**os.environ, "HOME": directory, "RAFT_NO_UPDATE_CHECK": "1"}
+        command = [sys.executable, str(ROOT / "raft.py"), "--json"]
+        response, records = events(command + ["--help"], env)
+        assert response.returncode == 0 and b"usage:" in stream(records, "stdout")
+        for arguments in [["invalid"], ["new", "--ttl", "no"], ["new", "--unknown"]]:
+            response, records = events(command + arguments, env)
+            assert response.returncode == 2 and records[0]["code"] == "usage", records
+        response, records = events(command + ["locations"], env)
+        assert response.returncode == 1 and records[0]["code"] == "operation", records
+        response, records = events(command + ["run", "--", "true"], env)
+        failures = [record for record in records if record["type"] == "error"]
+        assert response.returncode == 1 and len(failures) == 1, records
+        assert failures[0]["code"] == "operation" and "incus.json" in failures[0]["message"]
+        assert b"raft run: finished" in stream(records, "stderr"), records
+        plain = subprocess.run(
+            [sys.executable, str(ROOT / "raft.py"), "run", "--", "true"],
+            env=env,
+            capture_output=True,
+        )
+        assert plain.returncode == 1 and not plain.stdout
+        assert plain.stderr.startswith(b"raft run: ") and b"incus.json" in plain.stderr
+        inventory = Path(directory) / ".config/raft"
+        inventory.mkdir(parents=True)
+        (inventory / "incus.json").write_text('{"fixture":{"ssh":"offline.invalid","image":""}}')
+        response, records = events(command + ["locations"], env)
+        assert records[0] == {
+            "type": "data",
+            "value": {"location": "fixture", "default": True, "note": ""},
+        }
+        # Real subprocess descriptors, including binary bytes and more than a pipe
+        # buffer on both channels. Reconstruction must be exact and cannot hang.
+        program = """
+import os, subprocess, sys
+import raft_output
+out = raft_output.JsonOutput().start()
+subprocess.run([sys.executable, '-c', 'import os; os.write(1, bytes(range(256))*512); os.write(2,b"error"*32768)'], check=True)
+out.finish(7)
+sys.exit(7)
+"""
+        response, records = events(
+            [sys.executable, "-c", program], {**env, "PYTHONPATH": str(ROOT)}
+        )
+        assert response.returncode == 7
+        assert stream(records, "stdout") == bytes(range(256)) * 512
+        assert stream(records, "stderr") == b"error" * 32768
+        for number in (9, 15):
+            # The event and shell status must agree after real child termination.
+            signalled = f"""
+import subprocess, sys
+import raft, raft_output
+out = raft_output.JsonOutput().start()
+child = subprocess.run([sys.executable, '-c', 'import os, signal; os.kill(os.getpid(), {number})'])
+status = raft.exit_status(child.returncode)
+out.finish(status)
+sys.exit(status)
+"""
+            response, records = events(
+                [sys.executable, "-c", signalled], {**env, "PYTHONPATH": str(ROOT)}
+            )
+            assert response.returncode == 128 + number, records
+        parser = raft.parser()
+        parser.add_argument("--future-flag", action="store_true")
+        assert raft_output.candidates(parser, ["--future"]) == ["--future-flag"]
+        assert "--location" in raft_output.candidates(parser, ["new", "--lo"])
+        assert "set" in raft_output.candidates(parser, ["config", ""])
+        assert raft_output.candidates(parser, ["exec", "box", "--", ""]) == []
+        assert raft_output.candidates(parser, ["completions", "z"]) == ["zsh"]
+        for shell in ("bash", "zsh", "fish"):
+            script = subprocess.run(
+                [sys.executable, str(ROOT / "raft.py"), "completions", shell],
+                env=env,
+                capture_output=True,
+                check=True,
+            )
+            assert b"_complete" in script.stdout and not script.stderr
+        setup = raft_output.completions("bash")
+        completed = subprocess.run(
+            [
+                "bash",
+                "-c",
+                setup
+                + "\nraft() { "
+                + sys.executable
+                + " "
+                + str(ROOT / "raft.py")
+                + ' "$@"; }\nCOMP_WORDS=(raft config ""); COMP_CWORD=2; _raft_complete; printf "%s\\n" "${COMPREPLY[@]}"',
+            ],
+            env=env,
+            capture_output=True,
+            check=True,
+        )
+        assert "set" in completed.stdout.decode().splitlines(), completed
+    print("JSONL bytes, subprocess streams, structured errors, reports and Bash completion passed")
+
+
+def native(location):
+    spec = importlib.util.spec_from_file_location("live", ROOT / "deploy/verify-live.py")
+    live = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(live)
+    box = live.raft(
+        "new",
+        "--location",
+        location,
+        "--ttl",
+        "600",
+        "--disposable",
+        "--purpose",
+        "JSON output verification",
+        "--owner",
+        "verify-output",
+    ).stdout.strip()
+    try:
+        live.wait_running(box)
+        response, records = events(
+            [
+                sys.executable,
+                str(ROOT / "raft.py"),
+                "--json",
+                "exec",
+                box,
+                "--",
+                "sh",
+                "-c",
+                "printf hello; printf error >&2; exit 7",
+            ],
+            live.CONTROLLER_ENV,
+        )
+        assert response.returncode == 7
+        assert stream(records, "stdout") == b"hello" and stream(records, "stderr") == b"error", (
+            records
+        )
+        response, records = events(
+            [sys.executable, str(ROOT / "raft.py"), "--json", "info", box], live.CONTROLLER_ENV
+        )
+        assert records[0]["value"]["name"] == box.partition(":")[2], records
+        with tempfile.TemporaryDirectory(prefix="raft-output-artifacts-") as artifacts:
+            response, records = events(
+                [
+                    sys.executable,
+                    str(ROOT / "raft.py"),
+                    "--json",
+                    "run",
+                    "--location",
+                    location,
+                    "--purpose",
+                    "JSON artifact failure verification",
+                    "--owner",
+                    "verify-output",
+                    "--pull",
+                    "/missing-raft-artifacts",
+                    str(Path(artifacts) / "collected"),
+                    "--",
+                    "true",
+                ],
+                live.CONTROLLER_ENV,
+            )
+            assert response.returncode == 1, records
+            assert any(
+                record["type"] == "error"
+                and record["code"] == "operation"
+                and "could not collect /missing-raft-artifacts" in record["message"]
+                for record in records
+            ), records
+            assert b"cleanup confirmed" in stream(records, "stderr"), records
+        print(f"{location}: JSON exec, reports, run artifact failure and cleanup passed")
+    finally:
+        live.raft("destroy", box)
+
+
+if __name__ == "__main__":
+    cli = argparse.ArgumentParser(description=__doc__)
+    cli.add_argument("--location")
+    args = cli.parse_args()
+    verify()
+    if args.location:
+        native(args.location)

package/deploy/verify-release.py

@@ -68,12 +68,21 @@ def verify_npm():
             content = archive.extractfile(member).read()
             inspect(member.name, content)
             names.add(member.name)
-            if member.name in {"package/raft.py", "package/raft_files.py"}:
+            if member.name in {
+                "package/raft.py",
+                "package/raft_files.py",
+                "package/raft_run.py",
+                "package/raft_jobs.py",
+                "package/raft_output.py",
+            }:
                 if content != (ROOT / PurePosixPath(member.name).name).read_bytes():
                     raise ValueError("npm runtime differs from canonical Python source")
         required = {
             "package/raft.py",
             "package/raft_files.py",
+            "package/raft_run.py",
+            "package/raft_jobs.py",
+            "package/raft_output.py",
             "package/LICENSE",
             "package/deploy/deploy.py",
             "package/images/incus/build.sh",
@@ -105,12 +114,28 @@ def main():
                 raise ValueError("Unexpected non-file member: " + member.name)
             inspect(member.name, archive.extractfile(member).read())
         names = {PurePosixPath(member.name).parts[1] for member in members}
-        if not {"raft.py", "deploy", "docs", "images", "skills", "LICENSE"} <= names:
+        if (
+            not {
+                "raft.py",
+                "raft_files.py",
+                "raft_run.py",
+                "raft_jobs.py",
+                "raft_output.py",
+                "deploy",
+                "docs",
+                "images",
+                "skills",
+                "LICENSE",
+            }
+            <= names
+        ):
             raise ValueError("Source archive missing installation resources")
     with zipfile.ZipFile(wheels[0]) as archive:
         for name in archive.namelist():
             inspect(name, archive.read(name))
-        if not {"raft.py", "raft_files.py"} <= set(archive.namelist()):
+        if not {"raft.py", "raft_files.py", "raft_run.py", "raft_jobs.py", "raft_output.py"} <= set(
+            archive.namelist()
+        ):
             raise ValueError("Wheel missing CLI module")
     # A new environment avoids both editable checkouts and uv's cached same-version
     # tool environments. Run the installed entry point outside the source tree.
@@ -123,7 +148,11 @@ def main():
             check=True,
         )
         verify_cli(environment / "bin/raft", work)
-        subprocess.run([str(python), "-I", "-c", "import raft, raft_files"], cwd=work, check=True)
+        subprocess.run(
+            [str(python), "-I", "-c", "import raft, raft_files, raft_run, raft_jobs, raft_output"],
+            cwd=work,
+            check=True,
+        )
     verify_npm()
     print("Release archives and isolated wheel installation passed")