@synopackageland/cli 0.4.0 → 0.5.0

package/templates/native-go/backend/internal/helper/cohort_test.go

@@ -0,0 +1,372 @@
+package helper
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+)
+
+type operationFixture struct {
+	childFixture
+	stopped bool
+	pending bool
+	stuck   bool
+	leases  int
+}
+
+func (f *operationFixture) chunk(n uint32) error { f.pending = true; f.leases++; return nil }
+func (f *operationFixture) idle() (bool, error)  { return !f.pending, nil }
+func (f *operationFixture) stopRelay() {
+	f.stopped = true
+	if !f.stuck {
+		f.pending = false
+	}
+}
+func (f *operationFixture) relayStopped() bool { return f.stopped && !f.pending }
+
+func cohortFixture(t *testing.T) (*operationCohort, *NativeProvider, SessionCarriers, NativeActor, operationBinding, *[]*operationFixture) {
+	t.Helper()
+	p, _, _, carrier, now := nativeFixture()
+	actor, code := p.Resolve(context.Background(), carrier)
+	if code != "" {
+		t.Fatal(code)
+	}
+	i := Inventory{Package: "Fixture", Version: "1.0.0-0001", Protocol: 2, PayloadDigest: strings.Repeat("d", 64), RegistryDigest: strings.Repeat("e", 64)}
+	gate := newAdmission(Binding{Boot: "boot", Install: "install"})
+	gate.observe(gate.binding, 1, true, now)
+	children := new([]*operationFixture)
+	c := newOperationCohort(i, strings.Repeat("f", 64), 42, gate, func() bool { return true })
+	c.now = func() time.Time { return now }
+	c.startChild = func(a NativeActor, b operationBinding) (operationProcess, error) {
+		f := &operationFixture{childFixture: childFixture{exitOn: syscall.SIGTERM}}
+		*children = append(*children, f)
+		return f, nil
+	}
+	b := operationBinding{Inventory: i, Epoch: c.epoch, OperationID: fmt.Sprintf("%064x", 1), Nonce: fmt.Sprintf("%064x", 2), GrantID: strings.Repeat("1", 64), GrantRevision: 1}
+	return c, p, carrier, actor, b, children
+}
+func TestCohortPeerBindingActorReplayAndLimits(t *testing.T) {
+	for _, name := range []string{"peer", "uid", "inventory", "epoch", "zero-actor", "expired-actor", "root", "groups", "nonce", "revision"} {
+		t.Run(name, func(t *testing.T) {
+			c, p, carrier, a, b, children := cohortFixture(t)
+			peer := operationPeer{PID: 42, UID: 0}
+			switch name {
+			case "peer":
+				peer.PID = 43
+			case "uid":
+				peer.UID = 1
+			case "inventory":
+				b.Inventory.RegistryDigest = strings.Repeat("0", 64)
+			case "epoch":
+				b.Epoch = strings.Repeat("0", 64)
+			case "zero-actor":
+				a = NativeActor{}
+			case "expired-actor":
+				p.now = func() time.Time { return c.now().Add(3 * time.Second) }
+			case "root":
+				a.account.UID = 0
+			case "groups":
+				a.account.Groups = nil
+			case "nonce":
+				b.Nonce = "bad"
+			case "revision":
+				b.GrantRevision = 0
+			}
+			if c.start(context.Background(), peer, p, carrier, a, b) == "" || len(*children) != 0 {
+				t.Fatal("invalid work started")
+			}
+		})
+	}
+	c, p, carrier, a, b, children := cohortFixture(t)
+	if code := c.start(context.Background(), operationPeer{42, 0}, p, carrier, a, b); code != "" {
+		t.Fatal(code)
+	}
+	if code := c.start(context.Background(), operationPeer{42, 0}, p, carrier, a, b); code != ErrReplay {
+		t.Fatal(code)
+	}
+	b.OperationID = fmt.Sprintf("%064x", 3)
+	b.Nonce = fmt.Sprintf("%064x", 4)
+	if code := c.start(context.Background(), operationPeer{42, 0}, p, carrier, a, b); code != "" {
+		t.Fatal(code)
+	}
+	b.OperationID = fmt.Sprintf("%064x", 5)
+	b.Nonce = fmt.Sprintf("%064x", 6)
+	if code := c.start(context.Background(), operationPeer{42, 0}, p, carrier, a, b); code != ErrBusy || len(*children) != 2 {
+		t.Fatal(code)
+	}
+}
+func TestCohortChunkFenceAndCompletion(t *testing.T) {
+	c, p, carrier, a, b, children := cohortFixture(t)
+	peer := operationPeer{42, 0}
+	if code := c.start(context.Background(), peer, p, carrier, a, b); code != "" {
+		t.Fatal(code)
+	}
+	if _, err := c.proof(); err == nil {
+		t.Fatal("live cohort completed")
+	}
+	if code := c.chunk(context.Background(), peer, p, carrier, b, MaxChunkBytes); code != "" {
+		t.Fatal(code)
+	}
+	if code := c.chunk(context.Background(), peer, p, carrier, b, 1); code != ErrBusy {
+		t.Fatal(code)
+	}
+	c.gate.fence("fixture")
+	if code := c.chunk(context.Background(), peer, p, carrier, b, 1); code != ErrFenced {
+		t.Fatal(code)
+	}
+	if len((*children)[0].signals) != 0 {
+		t.Fatal("signal preceded durable fence/drain")
+	}
+	if err := c.drain(context.Background(), time.Millisecond); err != nil {
+		t.Fatal(err)
+	}
+	q, err := c.proof()
+	if err != nil || q.Started != 1 || q.Reaped != 1 || !q.RelaysStopped {
+		t.Fatal(q, err)
+	}
+	if !(*children)[0].stopped {
+		t.Fatal("relay survived")
+	}
+	if code := c.start(context.Background(), peer, p, carrier, a, b); code != ErrFenced {
+		t.Fatal(code)
+	}
+}
+func TestCohortUnknownReapAndBlockedRelayNeverComplete(t *testing.T) {
+	for _, kind := range []string{"ECHILD", "relay"} {
+		t.Run(kind, func(t *testing.T) {
+			c, p, carrier, a, b, children := cohortFixture(t)
+			if code := c.start(context.Background(), operationPeer{42, 0}, p, carrier, a, b); code != "" {
+				t.Fatal(code)
+			}
+			f := (*children)[0]
+			if kind == "ECHILD" {
+				f.err = syscall.ECHILD
+			} else {
+				f.stuck = true
+				f.pending = true
+			}
+			ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+			defer cancel()
+			if c.drain(ctx, time.Millisecond) == nil {
+				t.Fatal("unknown quiescence accepted")
+			}
+			if _, err := c.proof(); err == nil {
+				t.Fatal("invalid proof")
+			}
+			if kind == "ECHILD" && len(f.signals) != 0 {
+				t.Fatal("foreign child signalled")
+			}
+		})
+	}
+}
+func TestCohortRevalidationCancelsWithoutGrantRollback(t *testing.T) {
+	c, p, carrier, a, b, children := cohortFixture(t)
+	peer := operationPeer{42, 0}
+	if code := c.start(context.Background(), peer, p, carrier, a, b); code != "" {
+		t.Fatal(code)
+	}
+	p.accounts.(*fakeLocalAccount).value.Groups = []uint32{999}
+	if code := c.chunk(context.Background(), peer, p, carrier, b, 1); code != ErrStale {
+		t.Fatal(code)
+	}
+	if !(*children)[0].stopped {
+		t.Fatal("invalid identity left relay live")
+	}
+	if code := c.chunk(context.Background(), peer, p, carrier, b, 1); code != ErrCanceled {
+		t.Fatal(code)
+	}
+	if c.drain(context.Background(), time.Millisecond) != nil {
+		t.Fatal("drain failed")
+	}
+}
+
+func TestCohortPackageLimitGrantDrainAndReplayRetention(t *testing.T) {
+	c, p, carrier, a, b, children := cohortFixture(t)
+	peer := operationPeer{42, 0}
+	bindings := []operationBinding{}
+	for n := 0; n < 4; n++ {
+		subject := fmt.Sprintf("%064x", 100+n/2)
+		p.accounts.(*fakeLocalAccount).value.Principal.SubjectRef = subject
+		a, code := p.Resolve(context.Background(), carrier)
+		if code != "" {
+			t.Fatal(code)
+		}
+		b.OperationID = fmt.Sprintf("%064x", 10+n)
+		b.Nonce = fmt.Sprintf("%064x", 20+n)
+		b.GrantID = fmt.Sprintf("%064x", 30+n/2)
+		if code := c.start(context.Background(), peer, p, carrier, a, b); code != "" {
+			t.Fatal(code)
+		}
+		bindings = append(bindings, b)
+	}
+	p.accounts.(*fakeLocalAccount).value.Principal.SubjectRef = fmt.Sprintf("%064x", 500)
+	a, code := p.Resolve(context.Background(), carrier)
+	if code != "" {
+		t.Fatal(code)
+	}
+	b.OperationID = fmt.Sprintf("%064x", 99)
+	b.Nonce = fmt.Sprintf("%064x", 98)
+	if code := c.start(context.Background(), peer, p, carrier, a, b); code != ErrBusy {
+		t.Fatal(code)
+	}
+	if err := c.cancelGrant(context.Background(), bindings[0].GrantID, 1); err != nil {
+		t.Fatal(err)
+	}
+	if len(c.operations) != 2 || !(*children)[0].reaped || !(*children)[1].reaped || (*children)[2].stopped {
+		t.Fatal("per-grant drain affected wrong cohort")
+	}
+	// A completed ID remains burned even when slots become available.
+	if code := c.start(context.Background(), peer, p, carrier, a, bindings[0]); code != ErrReplay {
+		t.Fatal(code)
+	}
+	if err := c.drain(context.Background(), time.Millisecond); err != nil {
+		t.Fatal(err)
+	}
+	q, err := c.proof()
+	if err != nil || q.Reaped != 4 {
+		t.Fatal(q, err)
+	}
+}
+func TestCohortChunkBoundsAndForeignBinding(t *testing.T) {
+	for _, kind := range []string{"zero", "large", "total", "foreign", "helper-dead"} {
+		t.Run(kind, func(t *testing.T) {
+			c, p, carrier, a, b, _ := cohortFixture(t)
+			peer := operationPeer{42, 0}
+			if code := c.start(context.Background(), peer, p, carrier, a, b); code != "" {
+				t.Fatal(code)
+			}
+			n := uint32(1)
+			want := ErrLimit
+			switch kind {
+			case "zero":
+				n = 0
+			case "large":
+				n = MaxChunkBytes + 1
+			case "total":
+				c.operations[b.OperationID].bytes = MaxBoundedBytes
+			case "foreign":
+				b.GrantRevision++
+				want = ErrDenied
+			case "helper-dead":
+				c.helperLive = func() bool { return false }
+				want = ErrDenied
+			}
+			if code := c.chunk(context.Background(), peer, p, carrier, b, n); code != want {
+				t.Fatal(code)
+			}
+		})
+	}
+}
+func TestCohortAdmissionOrdersObserverFence(t *testing.T) {
+	c, p, carrier, a, b, _ := cohortFixture(t)
+	// Fence delivered during native lookup must win before spawn.
+	p.accounts.(*fakeLocalAccount).after = func() { c.gate.fence("native-race") }
+	if code := c.start(context.Background(), operationPeer{42, 0}, p, carrier, a, b); code != ErrFenced {
+		t.Fatal(code)
+	}
+}
+
+func TestCohortExpiredOperationStopsRelay(t *testing.T) {
+	c, p, carrier, a, b, children := cohortFixture(t)
+	peer := operationPeer{42, 0}
+	if code := c.start(context.Background(), peer, p, carrier, a, b); code != "" {
+		t.Fatal(code)
+	}
+	c.operations[b.OperationID].admittedAt = c.now().Add(-BoundedTimeout)
+	if code := c.chunk(context.Background(), peer, p, carrier, b, 1); code != ErrTimeout || !(*children)[0].stopped {
+		t.Fatal(code)
+	}
+}
+func TestCohortReplayWindowFullDoesNotEvict(t *testing.T) {
+	c, p, carrier, a, b, _ := cohortFixture(t)
+	for n := 0; n < MaxReplayEntries; n++ {
+		c.seen[fmt.Sprintf("op:%064x", n+1000)] = true
+		c.seen[fmt.Sprintf("nonce:%064x", n+2000)] = true
+	}
+	if code := c.start(context.Background(), operationPeer{42, 0}, p, carrier, a, b); code != ErrBusy {
+		t.Fatal(code)
+	}
+	if len(c.seen) != 2*MaxReplayEntries {
+		t.Fatal("replay history evicted")
+	}
+}
+func TestCohortPartialBootstrapRetainsChildUntilReaped(t *testing.T) {
+	c, p, carrier, a, b, children := cohortFixture(t)
+	factory := c.startChild
+	c.startChild = func(a NativeActor, b operationBinding) (operationProcess, error) {
+		child, _ := factory(a, b)
+		return child, ErrDenied
+	}
+	if code := c.start(context.Background(), operationPeer{42, 0}, p, carrier, a, b); code != ErrCleanupRequired {
+		t.Fatal(code)
+	}
+	if len(c.operations) != 1 || !(*children)[0].stopped {
+		t.Fatal("partial child lost")
+	}
+	if _, err := c.proof(); err == nil {
+		t.Fatal("unreaped child completed")
+	}
+	if c.drain(context.Background(), time.Millisecond) != nil {
+		t.Fatal("drain failed")
+	}
+	q, err := c.proof()
+	if err != nil || q.Reaped != 1 {
+		t.Fatal(q, err)
+	}
+}
+
+func TestCohortCompletionArtifactRejectsIncompleteProof(t *testing.T) {
+	for _, q := range []CohortCompletion{{Started: 2, Reaped: 1, RelaysStopped: true}, {Started: 2, Reaped: 2}, {Started: 0, Reaped: 0, RelaysStopped: true}, {Started: MaxReplayEntries + 1, Reaped: MaxReplayEntries + 1, RelaysStopped: true}} {
+		c, f := controlFixture(t)
+		r := Record{Inventory: c.Expected, Enabled: true, Lifecycle: testLifecycle("fenced")}
+		record(t, c.Client, r)
+		if err := cleanRuntime(c.Client, true); err != nil {
+			t.Fatal(err)
+		}
+		if completeRecord(c.Client, r.Lifecycle.Epoch, f.current, &q) == nil {
+			t.Fatal("invalid cohort completed", q)
+		}
+	}
+}
+
+func TestCohortCompletionArtifactRoundTrip(t *testing.T) {
+	c, f := controlFixture(t)
+	r := Record{Inventory: c.Expected, Enabled: true, Lifecycle: testLifecycle("fenced")}
+	record(t, c.Client, r)
+	if err := cleanRuntime(c.Client, true); err != nil {
+		t.Fatal(err)
+	}
+	q := CohortCompletion{Started: 2, Reaped: 2, RelaysStopped: true}
+	if err := completeRecord(c.Client, r.Lifecycle.Epoch, f.current, &q); err != nil {
+		t.Fatal(err)
+	}
+	current, err := readRecord(c.StateDir, c.Owner)
+	if err != nil {
+		t.Fatal(err)
+	}
+	proof, err := completionRecord(c.Client, current)
+	if err != nil || proof.Cohort == nil || *proof.Cohort != q {
+		t.Fatal(proof, err)
+	}
+}
+
+func TestCohortIdleOperationDeadlineCancelsBeforeNewLease(t *testing.T) {
+	c, p, carrier, a, b, children := cohortFixture(t)
+	if code := c.start(context.Background(), operationPeer{42, 0}, p, carrier, a, b); code != "" {
+		t.Fatal(code)
+	}
+	admitted := c.now()
+	c.now = func() time.Time { return admitted.Add(BoundedTimeout) }
+	if err := c.poll(); err != nil {
+		t.Fatal(err)
+	}
+	if !(*children)[0].stopped || len((*children)[0].signals) == 0 {
+		t.Fatal("idle operation survived deadline")
+	}
+	if _, err := c.proof(); err == nil {
+		t.Fatal("signal is not quiescence")
+	}
+}

package/templates/native-go/backend/internal/helper/control.go

@@ -9,6 +9,7 @@ import (
 	"net"
 	"os"
 	"path/filepath"
+	"strings"
 	"syscall"
 	"time"
 )
@@ -32,6 +33,8 @@ type Processes interface {
 type Controller struct {
 	Client
 	Processes Processes
+	Observe   Observation
+	Boot      func() (string, error)
 }
 
 func waitHealthy(ctx context.Context, probe func(context.Context) bool) error {
@@ -61,7 +64,7 @@ func writeProtected(dir, name string, value any) error {
 	}
 	defer os.Remove(f.Name())
 	mode := fs.FileMode(0644)
-	if name == "process.json" {
+	if name == "observer-diagnostic.json" || name == "process.json" || strings.HasPrefix(name, "completion-") {
 		mode = 0600
 	}
 	if err := f.Chmod(mode); err != nil {
@@ -120,7 +123,7 @@ func (c Controller) Control(ctx context.Context, action string) error {
 	if ctx.Err() != nil {
 		return ctx.Err()
 	}
-	if action != "enable" && action != "recover" && action != "teardown" && action != "disable" {
+	if action != "enable" && action != "recover" && action != "teardown" && action != "disable" && action != "repair" {
 		return errors.New("unsupported lifecycle action")
 	}
 	if !c.Expected.Valid() || c.Processes == nil {
@@ -145,50 +148,252 @@ func (c Controller) Control(ctx context.Context, action string) error {
 	if err != nil && !errors.Is(err, fs.ErrNotExist) {
 		return err
 	}
-	if action == "recover" && (err != nil || !r.Enabled) {
+	if errors.Is(err, fs.ErrNotExist) {
+		r = Record{Inventory: c.Expected}
+	}
+	if action == "repair" {
+		if err != nil {
+			return err
+		}
+		return c.repair(r)
+	}
+	starting := action == "enable" || action == "recover"
+	if action == "recover" && (err != nil || !r.Enabled || r.Inventory != c.Expected || r.Lifecycle == nil) {
 		return errors.New("explicit enable approval required")
 	}
-	if err != nil {
-		r = Record{Inventory: c.Expected}
+	// A stopped approval referencing a completion must have the private,
+	// digest-bound proof; missing/corrupt artifacts are not completion.
+	if starting && r.Lifecycle != nil && r.Lifecycle.Completion != "" {
+		if _, e := completionRecord(c.Client, r); e != nil {
+			return e
+		}
 	}
-	if action == "disable" {
-		r.Enabled = false
+	var next *Lifecycle
+	if starting {
+		if c.Observe == nil {
+			return errTrust
+		}
+		binding, e := c.Observe(ctx, c.Expected)
+		if e != nil {
+			return e
+		}
+		if action == "recover" && (r.Lifecycle.Binding.Boot != binding.Boot || r.Lifecycle.Binding.Install != binding.Install) {
+			return errors.New("installation continuity changed; enable approval required")
+		}
+		next, err = newLifecycle(binding)
+		if err != nil {
+			return err
+		}
+		if err := createLifetime(c.Client, next); err != nil {
+			return err
+		}
 	}
-	// Publish the durable gate first. A crashed/failed disable still refuses
-	// requests and exposes required cleanup; never report cleanup as successful.
-	r.Residual = true
-	if err := writeProtected(c.StateDir, "approval.json", r); err != nil {
+	// Preserve the owned identity before fencing lets the supervisor unlink it.
+	owned, ownedErr := readProcess(c.RunDir, c.Owner)
+	// Never hold policy.lock across Stop/ACK. Autonomous fencing shares only
+	// this short lock, so a controller holding control.lock cannot deadlock it.
+	if err := policyUpdate(c.Client, func(r *Record) error {
+		if action == "disable" {
+			r.Enabled = false
+		}
+		r.Residual = true
+		if r.Lifecycle != nil && r.Lifecycle.Phase != "stopped" {
+			r.Lifecycle.Phase = "fenced"
+			r.Lifecycle.Completion = ""
+			if r.Lifecycle.Reason == "" {
+				r.Lifecycle.Reason = "admin-stop"
+			}
+		}
+		return nil
+	}); err != nil {
 		return err
 	}
-	if err := c.teardown(); err != nil {
+	if ownedErr != nil && !errors.Is(ownedErr, fs.ErrNotExist) {
+		return ownedErr
+	}
+	if err := c.teardownFor(r, owned); err != nil {
 		return err
 	}
-	if action == "enable" || action == "recover" {
-		r = Record{Inventory: c.Expected, Enabled: true, Residual: true}
-		if err := writeProtected(c.StateDir, "approval.json", r); err != nil {
-			return err
-		}
+	if err := c.commitTeardown(r, owned, next); err != nil {
+		return err
+	}
+	if starting {
 		if err := c.Processes.Start(c.Expected); err != nil {
 			return err
 		}
+		return finalizeStart(c.Client, next.Epoch)
 	}
-	r.Residual = false
-	return writeProtected(c.StateDir, "approval.json", r)
+	return nil
+}
+
+// Keep the post-teardown policy commit separate from socket reconciliation.
+func (c Controller) commitTeardown(before Record, owned Process, next *Lifecycle) error {
+	observed, err := readRecord(c.StateDir, c.Owner)
+	if err != nil {
+		return err
+	}
+	if !sameTeardownEpoch(observed, before) {
+		return errTrust
+	}
+	published := ""
+	if observed.Lifecycle != nil {
+		published = observed.Lifecycle.Completion
+	}
+	// Rename makes completion visible before the owner's directory fsync and
+	// lock release. Wait only for that publisher; every other error is final.
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	last := errBusy
+	// A budget expiry keeps the last lock refusal; it never certifies success.
+	_ = waitHealthy(ctx, func(context.Context) bool {
+		last = policyUpdate(c.Client, func(r *Record) error {
+			if !sameTeardownEpoch(*r, before) || !runtimeAbsent(c.Client) {
+				return errTrust
+			}
+			if published != "" && (r.Lifecycle == nil || r.Lifecycle.Completion != published) {
+				return errTrust
+			}
+			if r.Lifecycle != nil && r.Lifecycle.Completion != "" {
+				proof, err := completionRecord(c.Client, *r)
+				if err != nil {
+					return err
+				}
+				if owned.PID > 1 {
+					if proof.Repair != nil || proof.Process != owned {
+						return errTrust
+					}
+				} else if before.Lifecycle == nil || before.Lifecycle.Phase != "stopped" {
+					return errTrust
+				}
+			}
+			if next != nil {
+				*r = Record{Inventory: c.Expected, Enabled: true, Residual: true, Lifecycle: next}
+			} else {
+				r.Residual = false
+				if r.Lifecycle != nil {
+					r.Lifecycle.Phase = "stopped"
+				}
+			}
+			return nil
+		})
+		return !errors.Is(last, errBusy)
+	})
+	return last
 }
 
 func (c Controller) teardown() error {
+	r, err := readRecord(c.StateDir, c.Owner)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return err
+	}
+	p, err := readProcess(c.RunDir, c.Owner)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return err
+	}
+	return c.teardownFor(r, p)
+}
+
+func (c Controller) teardownFor(before Record, owned Process) error {
+	r, err := readRecord(c.StateDir, c.Owner)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return err
+	}
+	if !sameTeardownEpoch(r, before) {
+		return errTrust
+	}
+	if r.Lifecycle != nil && r.Lifecycle.Completion != "" {
+		if owned.PID == 0 && before.Lifecycle != nil && before.Lifecycle.Phase != "stopped" {
+			return errTrust // no pre-fence process identity to reconcile this exit
+		}
+		if _, err := completionRecord(c.Client, r); err != nil {
+			return err
+		}
+	}
+	return c.teardownRecord(r, owned)
+}
+
+// Phase, completion and enabled/residual flags may change while the owning
+// supervisor exits; inventory, epoch, binding and optional lease must not.
+func sameTeardownEpoch(current, before Record) bool {
+	if current.Inventory != before.Inventory {
+		return false
+	}
+	a, b := current.Lifecycle, before.Lifecycle
+	if a == nil || b == nil {
+		return a == nil && b == nil
+	}
+	if a.Epoch != b.Epoch || a.Binding != b.Binding {
+		return false
+	}
+	if a.Lease == nil || b.Lease == nil {
+		return a.Lease == nil && b.Lease == nil
+	}
+	return *a.Lease == *b.Lease
+}
+
+// Reconcile runtime against the approval and pre-fence owned process witness.
+func (c Controller) teardownRecord(r Record, owned Process) error {
 	p, err := readProcess(c.RunDir, c.Owner)
 	if err != nil && !errors.Is(err, fs.ErrNotExist) {
 		return err
 	}
 	if err == nil {
-		if p.Inventory.Package != c.Expected.Package {
+		if p != owned || p.Inventory.Package != c.Expected.Package {
 			return errTrust
 		}
 		if err := c.Processes.Stop(p); err != nil {
+			if c.waitTeardownCompletion(r, owned) == nil {
+				return nil
+			}
 			return err
 		}
+	} else if owned.PID > 1 {
+		// The owner can unlink after our approval snapshot but before this read.
+		// Absence alone is never success, including the artifact-before-commit gap.
+		if c.waitTeardownCompletion(r, owned) == nil {
+			return nil
+		}
+		return errTrust
+	}
+	if errors.Is(err, fs.ErrNotExist) && r.Lifecycle != nil && r.Lifecycle.Phase == "fenced" {
+		// Runtime unlink followed by a crash is not committed completion. Only
+		// a verified new boot + absent /run artifacts permits explicit repair.
+		if c.Boot == nil || !runtimeAbsent(c.Client) {
+			return errTrust
+		}
+		boot, e := c.Boot()
+		if e != nil || boot == "" || boot == r.Lifecycle.Binding.Boot {
+			return errTrust
+		}
+	}
+	if err := cleanRuntime(c.Client, err == nil); err != nil {
+		if c.waitTeardownCompletion(r, owned) == nil {
+			return nil
+		}
+		return err
 	}
+	return nil
+}
+
+func (c Controller) waitTeardownCompletion(before Record, owned Process) error {
+	if before.Lifecycle == nil || owned.PID <= 1 || owned.Inventory != before.Inventory || owned.Boot != before.Lifecycle.Binding.Boot {
+		return errTrust
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	return waitHealthy(ctx, func(context.Context) bool {
+		r, err := readRecord(c.StateDir, c.Owner)
+		if err != nil || !sameTeardownEpoch(r, before) {
+			return false
+		}
+		proof, err := completionRecord(c.Client, r)
+		return err == nil && proof.Process == owned && runtimeAbsent(c.Client)
+	})
+}
+
+// cleanRuntime is shared by verified controller and owned supervisor cleanup.
+// The caller must first prove child teardown; it never signals a recorded PID.
+func cleanRuntime(c Client, hasProcess bool) error {
 	for _, name := range []string{"helper.sock", "supervisor.sock"} {
 		path := filepath.Join(c.RunDir, name)
 		st, e := os.Lstat(path)
@@ -212,7 +417,7 @@ func (c Controller) teardown() error {
 			return e
 		}
 	}
-	if err == nil {
+	if hasProcess {
 		return os.Remove(filepath.Join(c.RunDir, "process.json"))
 	}
 	return nil