@synopackageland/cli 0.3.10 → 0.4.1

package/dist/templates/native-go/backend/internal/helper/supervisor_linux_test.go

@@ -0,0 +1,913 @@
+package helper
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+)
+
+// This opt-in fixture uses a root-only, disposable Linux VM and an independently
+// cross-compiled cmd/helper. Never creates accounts or accesses a NAS.
+func linuxFixture(t *testing.T) LinuxProcesses {
+	t.Helper()
+	source := os.Getenv("M1C_HELPER_FIXTURE")
+	if source == "" {
+		t.Skip("requires disposable Linux root VM and M1C_HELPER_FIXTURE")
+	}
+	if os.Geteuid() != 0 {
+		t.Fatal("VM fixture requires root")
+	}
+	traces, _ := filepath.Glob("/tmp/m1h-trace-*")
+	for _, path := range traces {
+		if err := os.Remove(path); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if t.Name() == "TestLinuxHelperSelfExitIsReapedWithoutSignallingReusedPID" {
+		source = "/helper-self-exit"
+	}
+	if t.Name() == "TestLinuxControllerWaitsForRealCompletionPublisher" {
+		source = "/helper-held-completion"
+	}
+	digest, err := executableDigest(source)
+	if err != nil {
+		t.Fatal(err)
+	}
+	i := Inventory{Package: "root", Version: "1.0.0-0001", Protocol: 1, PayloadDigest: digest, RegistryDigest: strings.Repeat("b", 64)}
+	c := NewClient(i)
+
+	for _, dir := range []string{c.StateDir, c.RunDir} {
+		if err := os.Mkdir(dir, 0700); err != nil {
+			t.Fatal(err)
+		}
+		t.Cleanup(func() { os.RemoveAll(dir) })
+	}
+	t.Cleanup(func() {
+		r, e := readRecord(c.StateDir, 0)
+		b, _ := json.Marshal(r)
+		t.Logf("VM final lifecycle=%s readError=%v failed=%t", b, e, t.Failed())
+		if t.Failed() {
+			traces, _ := filepath.Glob("/tmp/m1h-trace-*")
+			for _, path := range traces {
+				b, _ := os.ReadFile(path)
+				t.Logf("VM trace %s: %s", path, b)
+			}
+		}
+	})
+	exe := filepath.Join(c.StateDir, "payload-"+digest)
+	b, err := os.ReadFile(source)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(exe, b, 0700); err != nil {
+		t.Fatal(err)
+	}
+	vmUnit(t, "start")
+	return LinuxProcesses{Client: c, Binary: exe}
+}
+func startVMHelper(t *testing.T, p LinuxProcesses) Process {
+	t.Helper()
+	ctl := Controller{Client: p.Client, Processes: p, Observe: observeLinux}
+	t.Cleanup(func() {
+		r, e := readRecord(p.StateDir, 0)
+		b, _ := json.Marshal(r)
+		t.Logf("VM pre-cleanup lifecycle=%s readError=%v failed=%t", b, e, t.Failed())
+		err := ctl.Control(context.Background(), "disable")
+		t.Logf("VM cleanup disable=%v", err)
+		if t.Failed() {
+			traces, _ := filepath.Glob("/tmp/m1h-trace-*")
+			for _, path := range traces {
+				b, _ := os.ReadFile(path)
+				t.Logf("VM trace %s: %s", path, b)
+			}
+		}
+	})
+	if err := ctl.Control(context.Background(), "enable"); err != nil {
+		t.Fatal(err)
+	}
+	record, err := readProcess(p.RunDir, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return record
+}
+func supervisedPID(t *testing.T, pid int) int {
+	t.Helper()
+	paths, _ := filepath.Glob("/proc/" + strconv.Itoa(pid) + "/task/*/children")
+	for _, path := range paths {
+		b, _ := os.ReadFile(path)
+		for _, s := range strings.Fields(string(b)) {
+			child, _ := strconv.Atoi(s)
+			if child > 1 {
+				return child
+			}
+		}
+	}
+	t.Fatal("supervisor has no child")
+	return 0
+}
+
+// Observation only: an absent process or zombie has exited. No signal is sent
+// to this PID, including if it has been reused. These fixtures run in a fresh VM.
+func waitPIDExit(t *testing.T, pid int) {
+	t.Helper()
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+	if waitHealthy(ctx, func(context.Context) bool {
+		b, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat")
+		if os.IsNotExist(err) {
+			return true
+		}
+		end := strings.LastIndex(string(b), ") ")
+		return err == nil && end >= 0 && strings.HasPrefix(string(b)[end+2:], "Z ")
+	}) != nil {
+		t.Fatal("owned process survived")
+	}
+}
+func TestLinuxActivationAlwaysUsesSupervisorAndNormalStop(t *testing.T) {
+	p := linuxFixture(t)
+	for n := 0; n < 20; n++ {
+		t.Run("cycle-"+strconv.Itoa(n), func(t *testing.T) {
+			proc := startVMHelper(t, p)
+			if proc.Mode != "supervised" {
+				t.Fatalf("activation did not select supervisor: %+v", proc)
+			}
+			child := supervisedPID(t, proc.PID)
+			if p.Status(context.Background()).State != "ready" {
+				t.Fatal("missing live handshake")
+			}
+			if err := (Controller{Client: p.Client, Processes: p, Observe: observeLinux}).Control(context.Background(), "disable"); err != nil {
+				t.Fatal(err)
+			}
+			waitPIDExit(t, child)
+			if err := p.CheckStopped(context.Background()); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}
+
+func TestLinuxHelperSelfExitIsReapedWithoutSignallingReusedPID(t *testing.T) {
+	p := linuxFixture(t)
+	proc := startVMHelper(t, p)
+	if proc.Mode != "supervised" {
+		t.Fatal("not supervised")
+	}
+	child := supervisedPID(t, proc.PID)
+	// The VM-only helper fixture exits itself when this root-owned marker exists.
+	if err := os.WriteFile(filepath.Join(p.RunDir, "fixture-exit"), nil, 0600); err != nil {
+		t.Fatal(err)
+	}
+	waitPIDExit(t, child)
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	if waitHealthy(ctx, func(context.Context) bool {
+		_, err := os.Stat("/proc/" + strconv.Itoa(child))
+		return os.IsNotExist(err)
+	}) != nil {
+		t.Fatal("supervisor did not reap")
+	}
+	if err := p.Stop(proc); err != nil {
+		t.Fatal(err)
+	}
+}
+func TestLinuxSupervisorDeathKillsHelperAndPreservesFailedCleanup(t *testing.T) {
+	p := linuxFixture(t)
+	record(t, p.Client, Record{Inventory: p.Expected, Enabled: true, Lifecycle: vmLifecycle(t, p)})
+	// Keep the supervisor as our own unreaped child; only its parent signals it.
+	cmd := exec.Command(p.Binary, "supervise")
+	if err := cmd.Start(); err != nil {
+		t.Fatal(err)
+	}
+	owned := &ownedLinuxChild{process: cmd.Process}
+	t.Cleanup(func() { stopChild(context.Background(), owned, time.Millisecond) })
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	err := waitHealthy(ctx, func(ctx context.Context) bool { return p.Status(ctx).State == "ready" })
+	cancel()
+	if err != nil {
+		t.Fatal(err)
+	}
+	proc, err := readProcess(p.RunDir, 0)
+	if err != nil || proc.Mode != "supervised" || proc.PID != cmd.Process.Pid {
+		t.Fatalf("%+v %v", proc, err)
+	}
+	child := supervisedPID(t, proc.PID)
+	if err := owned.signal(syscall.SIGKILL); err != nil {
+		t.Fatal(err)
+	}
+	waitPIDExit(t, child)
+	waitPIDExit(t, proc.PID)
+	ctl := Controller{Client: p.Client, Processes: p, Observe: observeLinux}
+	if ctl.Control(context.Background(), "teardown") == nil {
+		t.Fatal("lost supervisor falsely cleaned")
+	}
+	s := p.Status(context.Background())
+	if s.State != "failed" || s.Cleanup != "required" {
+		t.Fatal(s)
+	}
+	if ctl.Control(context.Background(), "disable") == nil {
+		t.Fatal("lost supervisor falsely disabled cleanly")
+	}
+	r, err := readRecord(p.StateDir, 0)
+	if err != nil || r.Enabled || !r.Residual {
+		t.Fatalf("%+v %v", r, err)
+	}
+	if _, err := os.Stat(filepath.Join(p.RunDir, "process.json")); err != nil {
+		t.Fatal("failure evidence removed")
+	}
+}
+func TestLinuxUnsupervisedRecordAndDirectServeRefuse(t *testing.T) {
+	p := linuxFixture(t)
+	proc := startVMHelper(t, p)
+	for _, mode := range []string{"", "unknown"} {
+		foreign := proc
+		foreign.Mode = mode
+		if p.Stop(foreign) == nil {
+			t.Fatal("unsupervised record accepted")
+		}
+	}
+	// Remove stopped sockets before testing the retired action, so a bind
+	// conflict cannot masquerade as refusal of unsupervised startup.
+	if err := (Controller{Client: p.Client, Processes: p, Observe: observeLinux}).Control(context.Background(), "teardown"); err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, p.Binary, "serve")
+	if cmd.Run() == nil || ctx.Err() != nil {
+		t.Fatal("direct unsupervised serve accepted")
+	}
+	if _, err := os.Stat(filepath.Join(p.RunDir, "helper.sock")); !os.IsNotExist(err) {
+		t.Fatal("direct serve opened socket")
+	}
+}
+
+func TestLinuxOwnedChildSelfExitAndECHILDRefusal(t *testing.T) {
+	for _, externallyReaped := range []bool{false, true} {
+		cmd := exec.Command("/bin/sh", "-c", "exit 0")
+		if err := cmd.Start(); err != nil {
+			t.Fatal(err)
+		}
+		child := &ownedLinuxChild{process: cmd.Process}
+		if externallyReaped {
+			if err := cmd.Wait(); err != nil {
+				t.Fatal(err)
+			}
+			if err := stopChild(context.Background(), child, time.Millisecond); !errors.Is(err, syscall.ECHILD) {
+				t.Fatal("ECHILD did not refuse", err)
+			}
+		} else {
+			ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+			err := waitHealthy(ctx, func(context.Context) bool { done, err := child.exited(); return done && err == nil })
+			cancel()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if err := stopChild(context.Background(), child, time.Millisecond); err != nil {
+				t.Fatal(err)
+			}
+			if child.signal(syscall.SIGTERM) == nil {
+				t.Fatal("reaped PID signalled")
+			}
+		}
+	}
+}
+func TestLinuxSupervisorIdentityConflictsAndSocketSubstitutionRefuse(t *testing.T) {
+	p := linuxFixture(t)
+	proc := startVMHelper(t, p)
+	for _, field := range []string{"boot", "start", "inode", "digest", "mode"} {
+		foreign := proc
+		switch field {
+		case "boot":
+			foreign.Boot = "reboot"
+		case "start":
+			foreign.Start = "reuse"
+		case "inode":
+			foreign.Inode++
+		case "digest":
+			foreign.Inventory.PayloadDigest = strings.Repeat("c", 64)
+		case "mode":
+			foreign.Mode = "unknown"
+		}
+		if p.Stop(foreign) == nil {
+			t.Fatal("foreign supervisor accepted", field)
+		}
+	}
+	socket := filepath.Join(p.RunDir, "supervisor.sock")
+	old := socket + ".owned"
+	if err := os.Rename(socket, old); err != nil {
+		t.Fatal(err)
+	}
+	fake, err := net.ListenUnix("unix", &net.UnixAddr{Name: socket, Net: "unix"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Chmod(socket, 0600); err != nil {
+		t.Fatal(err)
+	}
+	if p.Stop(proc) == nil {
+		t.Fatal("other root PID substituted supervisor")
+	}
+	fake.Close()
+	if err := os.Rename(old, socket); err != nil {
+		t.Fatal(err)
+	}
+	if p.Status(context.Background()).State != "ready" {
+		t.Fatal("refusal stopped real helper")
+	}
+	if err := p.Stop(proc); err != nil {
+		t.Fatal(err)
+	}
+}
+func TestLinuxReparentedChildRefusesBeforeSocket(t *testing.T) {
+	p := linuxFixture(t)
+	record(t, p.Client, Record{Inventory: p.Expected, Enabled: true, Lifecycle: vmLifecycle(t, p)})
+	cmd := exec.Command(p.Binary, "serve-child")
+	cmd.Env = []string{"PATH=/usr/sbin:/usr/bin:/sbin:/bin", "SYNOPKGLAND_PARENT=1"}
+	if cmd.Run() == nil {
+		t.Fatal("already reparented child served")
+	}
+	if _, err := os.Stat(filepath.Join(p.RunDir, "helper.sock")); !os.IsNotExist(err) {
+		t.Fatal("unsafe child opened socket")
+	}
+}
+
+func vmUnit(t *testing.T, action string) {
+	t.Helper()
+	if out, err := exec.Command("/bin/systemctl", action, "pkgctl-root.service").CombinedOutput(); err != nil {
+		t.Fatalf("systemctl %s: %v %s", action, err, out)
+	}
+}
+
+// Run the generated M1f hook as an unprivileged user against the real M1g
+// helper. The VM models management ordering, not DSM Package Center itself.
+func TestLinuxPrestopRequiresAdminTeardownAndPrestartNeverRecovers(t *testing.T) {
+	p := linuxFixture(t)
+	for _, dir := range []string{p.StateDir, p.RunDir} {
+		if err := os.Chmod(dir, 0755); err != nil {
+			t.Fatal(err)
+		}
+	}
+	proc := startVMHelper(t, p)
+	child := supervisedPID(t, proc.PID)
+	log := "/tmp/m1f-prestop-message"
+	if err := os.WriteFile(log, nil, 0600); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Chown(log, 65534, 65534); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { os.Remove(log) })
+	hook := func(action string, allow bool) {
+		t.Helper()
+		before, err := os.ReadFile(filepath.Join(p.StateDir, "approval.json"))
+		if err != nil {
+			t.Fatal(err)
+		}
+		if err := os.Truncate(log, 0); err != nil {
+			t.Fatal(err)
+		}
+		cmd := exec.Command("/var/packages/root/scripts/start-stop-status", action)
+		cmd.Env = append(os.Environ(), "SYNOPKG_TEMP_LOGFILE="+log)
+		cmd.SysProcAttr = &syscall.SysProcAttr{Credential: &syscall.Credential{Uid: 65534, Gid: 65534}}
+		out, err := cmd.CombinedOutput()
+		if (err == nil) != allow {
+			t.Fatalf("%s allow=%v: %v %s", action, allow, err, out)
+		}
+		if !allow {
+			message, err := os.ReadFile(log)
+			if err != nil || string(message) != string(out) || !strings.Contains(string(out), "helper_teardown") {
+				t.Fatalf("missing DSM refusal message: %v %s %s", err, out, message)
+			}
+		}
+		after, err := os.ReadFile(filepath.Join(p.StateDir, "approval.json"))
+		if err != nil || string(before) != string(after) {
+			t.Fatal("nonprivileged hook changed lifecycle latch", err)
+		}
+	}
+	hook("prestop", false)
+	if _, err := observeLinux(context.Background(), p.Expected); err != nil {
+		t.Fatal("prestop refusal changed active unit", err)
+	}
+	current, err := readProcess(p.RunDir, 0)
+	if err != nil || current != proc || p.Status(context.Background()).State != "ready" {
+		t.Fatal("prestop refusal stopped helper", err)
+	}
+	hook("prestart", true)
+	ctl := Controller{Client: p.Client, Processes: p, Observe: observeLinux}
+	if err := ctl.Control(context.Background(), "teardown"); err != nil {
+		t.Fatal(err)
+	}
+	waitPIDExit(t, child)
+	waitPIDExit(t, proc.PID)
+	hook("prestop", true)
+	vmUnit(t, "stop")
+	hook("stop", true)
+	vmUnit(t, "start")
+	hook("prestart", true)
+	hook("start", true)
+	if p.Status(context.Background()).State != "stopped" || !runtimeAbsent(p.Client) {
+		t.Fatal("package Start recovered integration")
+	}
+	stopped, err := readRecord(p.StateDir, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	incomplete := stopped
+	incomplete.Residual = true
+	record(t, p.Client, incomplete)
+	hook("prestop", false)
+	hook("prestart", true)
+	record(t, p.Client, stopped)
+	if err := ctl.Control(context.Background(), "recover"); err != nil {
+		t.Fatal(err)
+	}
+	recovered, err := readRecord(p.StateDir, 0)
+	if err != nil || recovered.Lifecycle.Epoch == stopped.Lifecycle.Epoch || p.Status(context.Background()).State != "ready" {
+		t.Fatal("explicit Recover failed", err)
+	}
+	if err := ctl.Control(context.Background(), "disable"); err != nil {
+		t.Fatal(err)
+	}
+	hook("prestop", true)
+	hook("prestart", true)
+	if p.Status(context.Background()).State != "not-enabled" || !runtimeAbsent(p.Client) {
+		t.Fatal("prestart cleared disabled gate")
+	}
+}
+
+func vmLifecycle(t *testing.T, p LinuxProcesses) *Lifecycle {
+	t.Helper()
+	b, err := observeLinux(context.Background(), p.Expected)
+	if err != nil {
+		t.Fatal(err)
+	}
+	l, err := newLifecycle(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := createLifetime(p.Client, l); err != nil {
+		t.Fatal(err)
+	}
+	return l
+}
+
+func TestLinuxUnitStopFencesBeforeLockAndPublishesCompletion(t *testing.T) {
+	p := linuxFixture(t)
+	proc := startVMHelper(t, p)
+	child := supervisedPID(t, proc.PID)
+	lock, err := os.OpenFile(filepath.Join(p.StateDir, "policy.lock"), os.O_RDWR, 0600)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer lock.Close()
+	if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX); err != nil {
+		t.Fatal(err)
+	}
+	vmUnit(t, "stop")
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	err = waitHealthy(ctx, func(context.Context) bool {
+		s := p.Status(context.Background())
+		return s.State == "failed" && s.Cleanup == "required"
+	})
+	cancel()
+	syscall.Flock(int(lock.Fd()), syscall.LOCK_UN)
+	if err != nil {
+		t.Fatal("memory fence waited for policy lock", err)
+	}
+	ctx, cancel = context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+	if waitHealthy(ctx, func(context.Context) bool {
+		r, e := readRecord(p.StateDir, 0)
+		return e == nil && !r.Residual && r.Lifecycle.Phase == "stopped" && r.Lifecycle.Completion != ""
+	}) != nil {
+		t.Fatal("completion missing")
+	}
+	r, _ := readRecord(p.StateDir, 0)
+	proof, e := completionRecord(p.Client, r)
+	if e != nil {
+		t.Fatal(e)
+	}
+	if proof.Process != proc {
+		t.Fatal("wrong completion identity", r)
+	}
+	waitPIDExit(t, child)
+	waitPIDExit(t, proc.PID)
+	if err := p.CheckStopped(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	ctl := Controller{Client: p.Client, Processes: p, Observe: observeLinux}
+	if ctl.Control(context.Background(), "recover") == nil {
+		t.Fatal("inactive unit recovered")
+	}
+	vmUnit(t, "start")
+	if p.Status(context.Background()).State != "stopped" {
+		t.Fatal("unit Start resurrected integration")
+	}
+	if err := ctl.Control(context.Background(), "recover"); err != nil {
+		t.Fatal(err)
+	}
+	next, _ := readRecord(p.StateDir, 0)
+	if next.Lifecycle.Epoch == r.Lifecycle.Epoch || p.Status(context.Background()).State != "ready" {
+		t.Fatal("explicit recovery failed")
+	}
+}
+func TestLinuxRapidUnitStopStartNeverClearsFence(t *testing.T) {
+	p := linuxFixture(t)
+	proc := startVMHelper(t, p)
+	vmUnit(t, "stop")
+	vmUnit(t, "start")
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+	if waitHealthy(ctx, func(context.Context) bool {
+		r, e := readRecord(p.StateDir, 0)
+		return e == nil && r.Lifecycle.Phase == "stopped" && !r.Residual
+	}) != nil {
+		t.Fatal("rapid transition missed")
+	}
+	waitPIDExit(t, proc.PID)
+	if p.Status(context.Background()).State != "stopped" {
+		t.Fatal("rapid Start resurrected")
+	}
+}
+func TestLinuxBusUnavailableRefusesActivation(t *testing.T) {
+	p := linuxFixture(t)
+	socket := busPath
+	if err := os.Rename(socket, socket+".fixture"); err != nil {
+		t.Fatal(err)
+	}
+	defer os.Rename(socket+".fixture", socket)
+	ctl := Controller{Client: p.Client, Processes: p, Observe: observeLinux}
+	if ctl.Control(context.Background(), "enable") == nil {
+		t.Fatal("activation without bus")
+	}
+	if !runtimeAbsent(p.Client) {
+		t.Fatal("unknown authority spawned runtime")
+	}
+}
+
+func TestLinuxRealOldPayloadReplayRefusesNewApproval(t *testing.T) {
+	p := linuxFixture(t)
+	_ = startVMHelper(t, p)
+	if err := (Controller{Client: p.Client, Processes: p, Observe: observeLinux}).Control(context.Background(), "teardown"); err != nil {
+		t.Fatal(err)
+	}
+	before, err := os.ReadFile(filepath.Join(p.StateDir, "approval.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	digest, err := executableDigest("/helper-legacy")
+	if err != nil {
+		t.Fatal(err)
+	}
+	path := filepath.Join(p.StateDir, "payload-"+digest)
+	b, err := os.ReadFile("/helper-legacy")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(path, b, 0700); err != nil {
+		t.Fatal(err)
+	}
+	for _, action := range []string{"enable", "recover", "supervise"} {
+		if out, err := exec.Command(path, action).CombinedOutput(); err == nil {
+			t.Fatalf("old payload replayed %s %s", action, out)
+		}
+	}
+	after, _ := os.ReadFile(filepath.Join(p.StateDir, "approval.json"))
+	if string(before) != string(after) || !runtimeAbsent(p.Client) {
+		t.Fatal("old payload changed latch")
+	}
+}
+func TestLinuxBusRestartFencesAndCompletesRuntime(t *testing.T) {
+	p := linuxFixture(t)
+	proc := startVMHelper(t, p)
+	if out, err := exec.Command("/bin/systemctl", "restart", "dbus.service").CombinedOutput(); err != nil {
+		t.Fatal(err, string(out))
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+	if waitHealthy(ctx, func(context.Context) bool {
+		r, e := readRecord(p.StateDir, 0)
+		return e == nil && r.Lifecycle.Phase == "stopped" && !r.Residual && r.Lifecycle.Completion != ""
+	}) != nil {
+		t.Fatal("bus loss did not complete")
+	}
+	waitPIDExit(t, proc.PID)
+	if p.Status(context.Background()).State != "stopped" {
+		t.Fatal("bus reconnection revived helper")
+	}
+}
+
+func TestLinuxLostSupervisorRepairThenExplicitRecover(t *testing.T) {
+	p := linuxFixture(t)
+	record(t, p.Client, Record{Inventory: p.Expected, Enabled: true, Lifecycle: vmLifecycle(t, p)})
+	cmd := exec.Command(p.Binary, "supervise")
+	if err := cmd.Start(); err != nil {
+		t.Fatal(err)
+	}
+	owned := &ownedLinuxChild{process: cmd.Process}
+	t.Cleanup(func() { stopChild(context.Background(), owned, time.Millisecond) })
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	err := waitHealthy(ctx, func(ctx context.Context) bool { return p.Status(ctx).State == "ready" })
+	cancel()
+	if err != nil {
+		t.Fatal(err)
+	}
+	proc, err := readProcess(p.RunDir, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	child := supervisedPID(t, proc.PID)
+	ctl := Controller{Client: p.Client, Processes: p, Observe: observeLinux}
+	if ctl.Control(context.Background(), "repair") == nil {
+		t.Fatal("repaired live supervisor")
+	}
+	if p.Status(context.Background()).State != "ready" {
+		t.Fatal("busy repair fenced active helper")
+	}
+	if err := owned.signal(syscall.SIGKILL); err != nil {
+		t.Fatal(err)
+	}
+	waitPIDExit(t, proc.PID)
+	waitPIDExit(t, child)
+	if ctl.Control(context.Background(), "recover") == nil {
+		t.Fatal("loss recovered without repair")
+	}
+	if err := ctl.Control(context.Background(), "repair"); err != nil {
+		t.Fatal(err)
+	}
+	repaired, err := readRecord(p.StateDir, 0)
+	if err != nil || repaired.Residual || repaired.Lifecycle.Completion == "" || p.Status(context.Background()).State != "stopped" {
+		t.Fatal("repair not committed", err)
+	}
+	proof, err := completionRecord(p.Client, repaired)
+	if err != nil || proof.Repair == nil || !runtimeAbsent(p.Client) {
+		t.Fatal("no repair proof", err)
+	}
+	if err := ctl.Control(context.Background(), "repair"); err != nil {
+		t.Fatal(err)
+	}
+	if err := ctl.Control(context.Background(), "recover"); err != nil {
+		t.Fatal(err)
+	}
+	next, _ := readRecord(p.StateDir, 0)
+	if next.Lifecycle.Epoch == repaired.Lifecycle.Epoch || p.Status(context.Background()).State != "ready" {
+		t.Fatal("explicit recovery did not start new epoch")
+	}
+	t.Cleanup(func() { ctl.Control(context.Background(), "disable") })
+	// A package/nonroot process cannot use the executable repair entry.
+	for _, dir := range []string{p.StateDir, p.RunDir} {
+		if err := os.Chmod(dir, 0755); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := os.Chmod(p.Binary, 0755); err != nil {
+		t.Fatal(err)
+	}
+	denied := exec.Command(p.Binary, "repair")
+	denied.SysProcAttr = &syscall.SysProcAttr{Credential: &syscall.Credential{Uid: 65534, Gid: 65534}}
+	output, err := denied.CombinedOutput()
+	if err == nil || !strings.Contains(string(output), "helper lifecycle refused") {
+		t.Fatal("package user did not reach euid refusal", err, string(output))
+	}
+
+}
+
+func TestLinuxInheritedLifetimePinsLockUntilLastHolderExit(t *testing.T) {
+	p := linuxFixture(t)
+	l := vmLifecycle(t, p)
+	lease, err := acquireLifetime(p.Client, l)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cmd := exec.Command("/bin/sleep", "60")
+	cmd.ExtraFiles = []*os.File{lease}
+	if err := cmd.Start(); err != nil {
+		t.Fatal(err)
+	}
+	child := &ownedLinuxChild{process: cmd.Process}
+	t.Cleanup(func() { stopChild(context.Background(), child, time.Millisecond) })
+	// Model supervisor loss while the inherited child is still alive. No
+	// LOCK_UN: closing that would release the shared flock prematurely.
+	lease.Close()
+	if got, err := acquireLifetime(p.Client, l); err == nil {
+		got.Close()
+		t.Fatal("parent close released live child's witness")
+	}
+	if err := stopChild(context.Background(), child, time.Millisecond); err != nil {
+		t.Fatal(err)
+	}
+	got, err := acquireLifetime(p.Client, l)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got.Close()
+}
+
+func TestLinuxPreR2PayloadReplayAndLateSupervisorRefuseRepairLatch(t *testing.T) {
+	p := linuxFixture(t)
+	record(t, p.Client, Record{Inventory: p.Expected, Enabled: true, Residual: true, Lifecycle: vmLifecycle(t, p)})
+	ctl := Controller{Client: p.Client, Processes: p, Observe: observeLinux}
+	// No process was started: the same lease also proves the startup crash gap.
+	if err := ctl.Control(context.Background(), "repair"); err != nil {
+		t.Fatal(err)
+	}
+	before, _ := os.ReadFile(filepath.Join(p.StateDir, "approval.json"))
+	if out, err := exec.Command(p.Binary, "supervise").CombinedOutput(); err == nil {
+		t.Fatal("late supervisor crossed repair latch", string(out))
+	}
+	for _, source := range []string{"/helper-pre-r2", "/helper-m1d"} {
+		digest, err := executableDigest(source)
+		if err != nil {
+			t.Fatal(err)
+		}
+		bytes, err := os.ReadFile(source)
+		if err != nil {
+			t.Fatal(err)
+		}
+		path := filepath.Join(p.StateDir, "payload-"+digest)
+		if err := os.WriteFile(path, bytes, 0700); err != nil {
+			t.Fatal(err)
+		}
+		for _, action := range []string{"enable", "recover", "supervise", "disable", "teardown"} {
+			if out, err := exec.Command(path, action).CombinedOutput(); err == nil {
+				t.Fatal("old payload crossed repair schema", source, action, string(out))
+			}
+		}
+	}
+	after, _ := os.ReadFile(filepath.Join(p.StateDir, "approval.json"))
+	if string(before) != string(after) || !runtimeAbsent(p.Client) {
+		t.Fatal("replay altered completion")
+	}
+}
+
+func TestLinuxActualM1dLostSupervisorResidueCannotGainRepairWitness(t *testing.T) {
+	p := linuxFixture(t)
+	digest, err := executableDigest("/helper-m1d")
+	if err != nil {
+		t.Fatal(err)
+	}
+	old := p
+	old.Expected.PayloadDigest = digest
+	old.Binary = filepath.Join(p.StateDir, "payload-"+digest)
+	bytes, err := os.ReadFile("/helper-m1d")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(old.Binary, bytes, 0700); err != nil {
+		t.Fatal(err)
+	}
+	record(t, p.Client, Record{Inventory: old.Expected, Enabled: true})
+	cmd := exec.Command(old.Binary, "supervise")
+	if err := cmd.Start(); err != nil {
+		t.Fatal(err)
+	}
+	owned := &ownedLinuxChild{process: cmd.Process}
+	t.Cleanup(func() { stopChild(context.Background(), owned, time.Millisecond) })
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	err = waitHealthy(ctx, func(ctx context.Context) bool { return old.Status(ctx).State == "ready" })
+	cancel()
+	if err != nil {
+		t.Fatal(err)
+	}
+	proc, err := readProcess(p.RunDir, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	child := supervisedPID(t, proc.PID)
+	if err := owned.signal(syscall.SIGKILL); err != nil {
+		t.Fatal(err)
+	}
+	waitPIDExit(t, proc.PID)
+	waitPIDExit(t, child)
+	// Mimic an upgraded controller; it must not create a new lock as purported
+	// historical proof, even though both observed PIDs have exited in this VM.
+	ctl := Controller{Client: p.Client, Processes: p, Observe: observeLinux}
+	if ctl.Control(context.Background(), "repair") == nil {
+		t.Fatal("upgraded helper repaired unwitnessed legacy residue")
+	}
+	for _, name := range []string{"helper.sock", "supervisor.sock", "process.json"} {
+		if _, err := os.Lstat(filepath.Join(p.RunDir, name)); err != nil {
+			t.Fatal("legacy evidence removed", name, err)
+		}
+	}
+	leases, _ := filepath.Glob(filepath.Join(p.StateDir, "lifetime-*.lock"))
+	if len(leases) != 0 {
+		t.Fatal("retroactively manufactured witness", leases)
+	}
+}
+
+func TestLinuxAdminTeardownSnapshotRace(t *testing.T) {
+	p := linuxFixture(t)
+	proc := startVMHelper(t, p)
+	child := supervisedPID(t, proc.PID)
+	ctl := Controller{Client: p.Client, Processes: p, Observe: observeLinux, Boot: linuxBoot}
+	// Capture the fenced snapshot inside the policy transaction, before the
+	// owner can complete it. Then deliberately let the real owner win the race.
+	var before Record
+	if err := policyUpdate(p.Client, func(r *Record) error {
+		r.Lifecycle.Phase = "fenced"
+		r.Lifecycle.Reason = "admin-stop"
+		r.Residual = true
+		before = *r
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+	if waitHealthy(ctx, func(context.Context) bool {
+		r, e := readRecord(p.StateDir, 0)
+		if e != nil {
+			return false
+		}
+		proof, e := completionRecord(p.Client, r)
+		return e == nil && proof.Process == proc && runtimeAbsent(p.Client)
+	}) != nil {
+		t.Fatal("owner did not commit autonomous admin-stop")
+	}
+	waitPIDExit(t, child)
+	waitPIDExit(t, proc.PID)
+	if err := ctl.teardownRecord(before, proc); err != nil {
+		t.Fatal("exact committed completion rejected after approval snapshot", err)
+	}
+}
+
+type pausedCompletionProcesses struct {
+	LinuxProcesses
+	visible string
+}
+
+func (p pausedCompletionProcesses) Stop(proc Process) error {
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+	if err := waitHealthy(ctx, func(context.Context) bool {
+		_, err := os.Stat(p.visible)
+		return err == nil
+	}); err != nil {
+		return err
+	}
+	// Exercise the real socket/identity path after autonomous runtime unlink.
+	return p.LinuxProcesses.Stop(proc)
+}
+
+func TestLinuxControllerWaitsForRealCompletionPublisher(t *testing.T) {
+	p := linuxFixture(t)
+	proc := startVMHelper(t, p)
+	child := supervisedPID(t, proc.PID)
+	hold := filepath.Join(p.StateDir, "fixture-hold-completion")
+	if err := os.WriteFile(hold, []byte("pause publisher after approval rename"), 0600); err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		os.Remove(hold) // disposable VM fixture only; never a NAS operation
+		waitPIDExit(t, child)
+		waitPIDExit(t, proc.PID)
+	}()
+	result := make(chan error, 1)
+	go func() {
+		processes := pausedCompletionProcesses{LinuxProcesses: p, visible: filepath.Join(p.StateDir, "fixture-completion-visible")}
+		result <- (Controller{Client: p.Client, Processes: processes, Observe: observeLinux, Boot: linuxBoot}).Control(context.Background(), "teardown")
+	}()
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+	if waitHealthy(ctx, func(context.Context) bool {
+		_, visible := os.Stat(filepath.Join(p.StateDir, "fixture-completion-visible"))
+		r, e := readRecord(p.StateDir, 0)
+		if visible != nil || e != nil {
+			return false
+		}
+		proof, e := completionRecord(p.Client, r)
+		return e == nil && proof.Process == proc && runtimeAbsent(p.Client)
+	}) != nil {
+		t.Fatal("real supervisor did not publish while holding policy.lock")
+	}
+	select {
+	case err := <-result:
+		t.Fatal("controller returned before publisher unlocked", err)
+	case <-time.After(100 * time.Millisecond):
+	}
+	if err := os.Remove(hold); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case err := <-result:
+		if err != nil {
+			t.Fatal("real committed completion refused after publisher unlock", err)
+		}
+	case <-ctx.Done():
+		t.Fatal("controller did not finish after publisher unlock")
+	}
+	if err := p.CheckStopped(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+}