@synopackageland/cli 0.4.0 → 0.4.1

package/templates/native-go/backend/internal/helper/operation_linux_test.go

@@ -0,0 +1,528 @@
+package helper
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"slices"
+	"strconv"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+)
+
+// Runs only in the disposable NIC-free root VM with a separately built payload.
+func TestLinuxOperationCredentialsChunkBindingAndDrain(t *testing.T) {
+	p := linuxFixture(t)
+	proc := startVMHelper(t, p)
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+	// The test process is the direct parent here; the full supervisor fixture
+	// below independently verifies production parent identity and lifetime binding.
+	a := NativeActor{valid: true, account: LocalAccount{UID: 1001, GID: 100, Groups: []uint32{101, 105}}}
+	b := operationBinding{Inventory: p.Expected, Epoch: "fixture", OperationID: fmt.Sprintf("%064x", 1)}
+	if _, err := startLinuxOperation(p.Client, p.Binary, a, b, nil); err == nil {
+		t.Fatal("non-supervisor operation parent accepted")
+	}
+	if err := p.Stop(proc); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestLinuxOperationBootstrapFixture(t *testing.T) {
+	if os.Getenv("M2_OPERATION_VM") != "1" {
+		t.Skip("root VM fixture")
+	}
+	p := operationVMFixture(t)
+	proc := startVMHelper(t, p)
+	// The fixture-only helper starts two operation children from the locked
+	// supervisor loop and writes root-owned observation records, never NAS data.
+	marker := p.RunDir + "/fixture-operations"
+	if err := os.WriteFile(marker, nil, 0600); err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+	if waitHealthy(ctx, func(context.Context) bool {
+		_, err := os.Stat(p.RunDir + "/fixture-operations-ready")
+		return err == nil
+	}) != nil {
+		t.Fatal("cohort did not bootstrap")
+	}
+	children := operationChildren(t, proc.PID)
+	if len(children) != 3 {
+		t.Fatalf("supervisor must directly own helper plus two operations: %v", children)
+	}
+	for _, pid := range children {
+		if pid == supervisedPIDByMode(t, proc.PID, "serve-child") {
+			continue
+		}
+		status, err := os.ReadFile(fmt.Sprintf("/proc/%d/status", pid))
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !statusHasCredentials(status, 1001, 100, []uint32{101, 105}) {
+			t.Fatalf("credential mismatch: %s", status)
+		}
+	}
+	// A blocked output pipe/relay must close before stop ACK and completion.
+	vmUnit(t, "stop")
+	if waitHealthy(ctx, func(context.Context) bool {
+		r, err := readRecord(p.StateDir, 0)
+		return err == nil && r.Lifecycle.Phase == "stopped"
+	}) != nil {
+		t.Fatal("cohort stop failed")
+	}
+	r, err := readRecord(p.StateDir, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	proof, err := completionRecord(p.Client, r)
+	if err != nil || proof.Cohort == nil || proof.Cohort.Started != 2 || proof.Cohort.Reaped != 2 || !proof.Cohort.RelaysStopped {
+		t.Fatal(proof, err)
+	}
+	for _, pid := range children {
+		if _, err := os.Stat(fmt.Sprintf("/proc/%d", pid)); !os.IsNotExist(err) {
+			t.Fatal("child not reaped", pid)
+		}
+	}
+}
+func operationChildren(t *testing.T, pid int) []int {
+	t.Helper()
+	return processChildren(pid)
+}
+func supervisedPIDByMode(t *testing.T, pid int, mode string) int {
+	t.Helper()
+	for _, child := range processChildren(pid) {
+		b, _ := os.ReadFile(fmt.Sprintf("/proc/%d/cmdline", child))
+		if slices.Contains(splitCommand(b), mode) {
+			return child
+		}
+	}
+	t.Fatal("missing child mode", mode)
+	return 0
+}
+func TestLinuxOperationParentDeathAfterDrop(t *testing.T) {
+	if os.Getenv("M2_OPERATION_VM") != "1" {
+		t.Skip("root VM fixture")
+	}
+	p := operationVMFixture(t)
+	record(t, p.Client, Record{Inventory: p.Expected, Enabled: true, Lifecycle: vmLifecycle(t, p)})
+	owned := startOwnedSupervisor(t, p)
+	proc, err := readProcess(p.RunDir, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(p.RunDir+"/fixture-pause-operation", nil, 0600); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(p.RunDir+"/fixture-operations", nil, 0600); err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+	if waitHealthy(ctx, func(context.Context) bool {
+		_, err := os.Stat(p.RunDir + "/fixture-operations-ready")
+		return err == nil
+	}) != nil {
+		t.Fatal("not ready")
+	}
+	children := processChildren(proc.PID)
+	if len(children) != 3 {
+		t.Fatal(children)
+	}
+	if err := owned.signal(syscall.SIGKILL); err != nil {
+		t.Fatal(err)
+	}
+	for _, pid := range children {
+		waitPIDExit(t, pid)
+	}
+	waitPIDExit(t, proc.PID)
+	r, err := readRecord(p.StateDir, 0)
+	if err != nil || r.Lifecycle.Completion != "" {
+		t.Fatal("crash fabricated completion", r, err)
+	}
+}
+func TestLinuxOperationHelperCrashDrainsCohort(t *testing.T) {
+	if os.Getenv("M2_OPERATION_VM") != "1" {
+		t.Skip("root VM fixture")
+	}
+	p := operationVMFixture(t)
+	_ = startVMHelper(t, p)
+	if err := os.WriteFile(p.RunDir+"/fixture-operations", nil, 0600); err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+	if waitHealthy(ctx, func(context.Context) bool {
+		_, err := os.Stat(p.RunDir + "/fixture-operations-ready")
+		return err == nil
+	}) != nil {
+		t.Fatal("not ready")
+	}
+	// Fixture-only owner-loop hook signals its own unreaped helper.
+	if err := os.WriteFile(p.RunDir+"/fixture-kill-helper", nil, 0600); err != nil {
+		t.Fatal(err)
+	}
+	if waitHealthy(ctx, func(context.Context) bool {
+		r, err := readRecord(p.StateDir, 0)
+		return err == nil && r.Lifecycle.Phase == "stopped"
+	}) != nil {
+		t.Fatal("helper death did not fence/drain")
+	}
+	r, _ := readRecord(p.StateDir, 0)
+	proof, err := completionRecord(p.Client, r)
+	if err != nil || proof.Cohort == nil || proof.Cohort.Reaped != 2 {
+		t.Fatal(proof, err)
+	}
+}
+
+// Verify binary relay backpressure cancellation with a real socket and pipe.
+func TestLinuxOperationRelayCancellation(t *testing.T) {
+	if os.Getenv("M2_OPERATION_VM") != "1" {
+		t.Skip("root VM fixture")
+	}
+	r, w, err := os.Pipe()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer w.Close()
+	left, right := net.Pipe()
+	defer right.Close()
+	child := newLinuxOperationRelay(nil, nil, r, left)
+	if err := child.beginRelay(MaxChunkBytes); err != nil {
+		t.Fatal(err)
+	}
+	go func() { _, _ = io.CopyN(w, zeroReader{}, MaxChunkBytes) }()
+	child.stopRelay()
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	if waitHealthy(ctx, func(context.Context) bool { return child.relayStopped() }) != nil {
+		t.Fatal("relay survived close")
+	}
+}
+
+func operationVMFixture(t *testing.T) LinuxProcesses {
+	t.Helper()
+	t.Setenv("M1C_HELPER_FIXTURE", os.Getenv("M2_OPERATION_FIXTURE"))
+	return linuxFixture(t)
+}
+func startOwnedSupervisor(t *testing.T, p LinuxProcesses) *ownedLinuxChild {
+	t.Helper()
+	cmd := exec.Command(p.Binary, "supervise")
+	if err := cmd.Start(); err != nil {
+		t.Fatal(err)
+	}
+	child := &ownedLinuxChild{process: cmd.Process}
+	t.Cleanup(func() {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+		defer cancel()
+		_ = stopChild(ctx, child, time.Millisecond)
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+	if waitHealthy(ctx, func(ctx context.Context) bool { return p.Status(ctx).State == "ready" }) != nil {
+		t.Fatal("supervisor did not start")
+	}
+	return child
+}
+func processChildren(pid int) []int {
+	result := []int{}
+	paths, _ := filepath.Glob(fmt.Sprintf("/proc/%d/task/*/children", pid))
+	for _, p := range paths {
+		b, _ := os.ReadFile(p)
+		for _, s := range strings.Fields(string(b)) {
+			n, _ := strconv.Atoi(s)
+			if n > 1 {
+				result = append(result, n)
+			}
+		}
+	}
+	return result
+}
+func splitCommand(b []byte) []string { return strings.Split(string(b), "\x00") }
+func statusHasCredentials(b []byte, uid, gid uint32, groups []uint32) bool {
+	values := map[string][]string{}
+	for _, line := range strings.Split(string(b), "\n") {
+		k, v, ok := strings.Cut(line, ":")
+		if ok {
+			values[k] = strings.Fields(v)
+		}
+	}
+	for _, field := range []struct {
+		name string
+		id   uint32
+	}{{"Uid", uid}, {"Gid", gid}} {
+		v := values[field.name]
+		if len(v) != 4 {
+			return false
+		}
+		for _, s := range v {
+			if s != fmt.Sprint(field.id) {
+				return false
+			}
+		}
+	}
+	got := []uint32{}
+	for _, s := range values["Groups"] {
+		n, err := strconv.ParseUint(s, 10, 32)
+		if err != nil {
+			return false
+		}
+		got = append(got, uint32(n))
+	}
+	slices.Sort(got)
+	return slices.Equal(got, groups)
+}
+
+func TestLinuxOperationForeignRootPeerAndDirectModeRefuse(t *testing.T) {
+	p := linuxFixture(t)
+	proc := startVMHelper(t, p)
+	conn, err := net.DialUnix("unix", nil, &net.UnixAddr{Name: p.RunDir + "/supervisor.sock", Net: "unix"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer conn.Close()
+	conn.SetDeadline(time.Now().Add(time.Second))
+	nonce := strings.Repeat("a", 64)
+	if err := json.NewEncoder(conn).Encode(supervisorRequest{Process: proc, Op: "admit", Nonce: nonce}); err != nil {
+		t.Fatal(err)
+	}
+	var reply Response
+	if json.NewDecoder(conn).Decode(&reply) != nil || reply.OK || reply.Error != "fenced" {
+		t.Fatal("foreign root child admitted", reply)
+	}
+	cmd := exec.Command(p.Binary, "operation-child")
+	cmd.Env = []string{"SYNOPKGLAND_PARENT=" + strconv.Itoa(os.Getpid())}
+	if cmd.Run() == nil {
+		t.Fatal("foreign operation bootstrap accepted")
+	}
+}
+func TestLinuxOperationRealBinaryRelayAndOneLease(t *testing.T) {
+	if os.Getenv("M2_OPERATION_VM") != "1" {
+		t.Skip("root VM fixture")
+	}
+	r, w, err := os.Pipe()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer w.Close()
+	left, right := net.Pipe()
+	defer right.Close()
+	child := newLinuxOperationRelay(nil, nil, r, left)
+	if err := child.beginRelay(64); err != nil {
+		t.Fatal(err)
+	}
+	if err := child.beginRelay(64); err == nil {
+		t.Fatal("parallel lease accepted")
+	}
+	expected := bytes.Repeat([]byte{0x5a}, 64)
+	go func() { _, _ = w.Write(expected) }()
+	actual := make([]byte, 64)
+	right.SetReadDeadline(time.Now().Add(time.Second))
+	if _, err := io.ReadFull(right, actual); err != nil || !bytes.Equal(actual, expected) {
+		t.Fatal("binary bytes lost", err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	if waitHealthy(ctx, func(context.Context) bool { idle, err := child.idle(); return idle && err == nil }) != nil {
+		t.Fatal("relay never finished")
+	}
+	child.stopRelay()
+	if waitHealthy(ctx, func(context.Context) bool { return child.relayStopped() }) != nil {
+		t.Fatal("relay or output watcher survived close")
+	}
+}
+
+func TestLinuxOperationBusyPolicyFenceDefersSignalsAndRepair(t *testing.T) {
+	if os.Getenv("M2_OPERATION_VM") != "1" {
+		t.Skip("root VM fixture")
+	}
+	p := operationVMFixture(t)
+	proc := startVMHelper(t, p)
+	if err := os.WriteFile(p.RunDir+"/fixture-pause-operation", nil, 0600); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(p.RunDir+"/fixture-operations", nil, 0600); err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+	if waitHealthy(ctx, func(context.Context) bool {
+		_, err := os.Stat(p.RunDir + "/fixture-operations-ready")
+		return err == nil
+	}) != nil {
+		t.Fatal("not ready")
+	}
+	children := processChildren(proc.PID)
+	r, err := readRecord(p.StateDir, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// All operation children hold the same inherited epoch witness.
+	if lease, err := acquireLifetime(p.Client, r.Lifecycle); err == nil {
+		lease.Close()
+		t.Fatal("live cohort did not hold repair witness")
+	}
+	lock, err := os.OpenFile(p.StateDir+"/policy.lock", os.O_RDWR, 0600)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer lock.Close()
+	if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX); err != nil {
+		t.Fatal(err)
+	}
+	vmUnit(t, "stop")
+	time.Sleep(150 * time.Millisecond)
+	current, err := readRecord(p.StateDir, 0)
+	if err != nil || current.Lifecycle.Phase != "running" {
+		t.Fatal("durable fence ignored lock", current, err)
+	}
+	for _, pid := range children {
+		b, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid))
+		if err != nil || strings.Contains(string(b), ") Z ") {
+			t.Fatal("child signalled before durable fence", pid, err)
+		}
+	}
+	if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_UN); err != nil {
+		t.Fatal(err)
+	}
+	if waitHealthy(ctx, func(context.Context) bool {
+		r, err := readRecord(p.StateDir, 0)
+		return err == nil && r.Lifecycle.Phase == "stopped"
+	}) != nil {
+		t.Fatal("durable fence did not drain")
+	}
+	current, _ = readRecord(p.StateDir, 0)
+	proof, err := completionRecord(p.Client, current)
+	if err != nil || proof.Cohort == nil || !proof.Cohort.valid() {
+		t.Fatal(proof, err)
+	}
+}
+
+func TestLinuxOperationFenceRetryBudgetRetainsOwnerUntilExplicitStop(t *testing.T) {
+	if os.Getenv("M2_OPERATION_VM") != "1" {
+		t.Skip("root VM fixture")
+	}
+	p := operationVMFixture(t)
+	proc := startVMHelper(t, p)
+	for _, name := range []string{"fixture-pause-operation", "fixture-operations"} {
+		if err := os.WriteFile(p.RunDir+"/"+name, nil, 0600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second)
+	defer cancel()
+	if waitHealthy(ctx, func(context.Context) bool {
+		_, err := os.Stat(p.RunDir + "/fixture-operations-ready")
+		return err == nil
+	}) != nil {
+		t.Fatal("not ready")
+	}
+	children := processChildren(proc.PID)
+	lock, err := os.OpenFile(p.StateDir+"/policy.lock", os.O_RDWR, 0600)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer lock.Close()
+	if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX); err != nil {
+		t.Fatal(err)
+	}
+	vmUnit(t, "stop")
+	// Exceed the one-second retry budget. Pipe EOF cannot terminate paused ops.
+	time.Sleep(1200 * time.Millisecond)
+	for _, pid := range append(children, proc.PID) {
+		b, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid))
+		if err != nil || strings.Contains(string(b), ") Z ") {
+			t.Fatal("budget expiry signalled or exited owner/child", pid, err)
+		}
+	}
+	r, err := readRecord(p.StateDir, 0)
+	if err != nil || r.Lifecycle.Phase != "running" || r.Lifecycle.Completion != "" {
+		t.Fatal("uncommitted fence completed", r, err)
+	}
+	if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_UN); err != nil {
+		t.Fatal(err)
+	}
+	// Timeout must not restart an unbounded automatic retry loop.
+	time.Sleep(100 * time.Millisecond)
+	r, err = readRecord(p.StateDir, 0)
+	if err != nil || r.Lifecycle.Phase != "running" {
+		t.Fatal("automatic retries restarted", r, err)
+	}
+	if err := (Controller{Client: p.Client, Processes: p, Observe: observeLinux, Boot: linuxBoot}).Control(context.Background(), "teardown"); err != nil {
+		t.Fatal("explicit durable stop refused", err)
+	}
+	for _, pid := range children {
+		waitPIDExit(t, pid)
+	}
+	if err := p.CheckStopped(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+}
+func TestLinuxOperationControllerWaitsForCohortCompletionPublisher(t *testing.T) {
+	if os.Getenv("M2_OPERATION_VM") != "1" {
+		t.Skip("root VM fixture")
+	}
+	t.Setenv("M2_OPERATION_FIXTURE", os.Getenv("M2_OPERATION_HELD_FIXTURE"))
+	p := operationVMFixture(t)
+	proc := startVMHelper(t, p)
+	if err := os.WriteFile(p.RunDir+"/fixture-operations", nil, 0600); err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second)
+	defer cancel()
+	if waitHealthy(ctx, func(context.Context) bool {
+		_, err := os.Stat(p.RunDir + "/fixture-operations-ready")
+		return err == nil
+	}) != nil {
+		t.Fatal("not ready")
+	}
+	hold := p.StateDir + "/fixture-hold-completion"
+	if err := os.WriteFile(hold, nil, 0600); err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(hold)
+	result := make(chan error, 1)
+	go func() {
+		processes := pausedCompletionProcesses{LinuxProcesses: p, visible: p.StateDir + "/fixture-completion-visible"}
+		result <- (Controller{Client: p.Client, Processes: processes, Observe: observeLinux, Boot: linuxBoot}).Control(context.Background(), "teardown")
+	}()
+	if waitHealthy(ctx, func(context.Context) bool {
+		r, err := readRecord(p.StateDir, 0)
+		if err != nil {
+			return false
+		}
+		proof, err := completionRecord(p.Client, r)
+		return err == nil && proof.Process == proc && proof.Cohort != nil && proof.Cohort.Reaped == 2 && proof.Cohort.RelaysStopped
+	}) != nil {
+		t.Fatal("cohort completion not visible")
+	}
+	select {
+	case err := <-result:
+		t.Fatal("controller bypassed publisher lock", err)
+	case <-time.After(100 * time.Millisecond):
+	}
+	if err := os.Remove(hold); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case err := <-result:
+		if err != nil {
+			t.Fatal("exact cohort proof refused after unlock", err)
+		}
+	case <-ctx.Done():
+		t.Fatal("controller remained blocked")
+	}
+	if err := p.CheckStopped(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+}

package/templates/native-go/backend/internal/helper/process_linux.go

@@ -148,6 +148,8 @@ func RunLinux(base Inventory, action string) error {
 	switch action {
 	case "supervise":
 		return superviseLinux(c, exe)
+	case "operation-child":
+		return operationChildLinux(c)
 	case "serve-child":
 		parent, err := strconv.Atoi(os.Getenv("SYNOPKGLAND_PARENT"))
 		if err != nil {
@@ -161,7 +163,15 @@ func RunLinux(base Inventory, action string) error {
 		if _, err := inspectSupervisor(parent, base); err != nil {
 			return err
 		}
+		if err := inheritLifetime(c); err != nil {
+			return err
+		}
 		return serveLinux(c)
 	}
-	return (Controller{Client: c, Processes: LinuxProcesses{Client: c, Binary: exe}}).Control(context.Background(), action)
+	return (Controller{Client: c, Processes: LinuxProcesses{Client: c, Binary: exe}, Observe: observeLinux, Boot: linuxBoot}).Control(context.Background(), action)
+}
+
+func linuxBoot() (string, error) {
+	b, err := os.ReadFile("/proc/sys/kernel/random/boot_id")
+	return strings.TrimSpace(string(b)), err
 }