devrites 5.2.4 → 5.4.0

package/engine/internal/parallel/ops.go

@@ -7,6 +7,9 @@ import (
 	"path/filepath"
 	"sort"
 	"strings"
+
+	"github.com/devrites/devrites/internal/devritespaths"
+	"github.com/devrites/devrites/internal/state"
 )
 
 // Test seams: indirection over the real primitives so discriminating tests
@@ -14,6 +17,7 @@ import (
 var (
 	writeLease     = WriteLease
 	removeWorktree = worktreeRemove
+	salvage        = salvageSlice
 	warnf          = func(format string, args ...any) {
 		fmt.Fprintf(os.Stderr, "warning: parallel: "+format+"\n", args...)
 	}
@@ -29,11 +33,35 @@ type CreateOpts struct {
 	Slices  []SlicePaths
 }
 
+// withLeaseLock serializes lease read-modify-write across engine processes:
+// the host may issue record-green/integrate calls concurrently, and a
+// lock-free pair of them would lose one update.
+func withLeaseLock(repoRoot, slug string, fn func() error) error {
+	if err := validateSlug(slug); err != nil {
+		return err
+	}
+	return state.WithFeatureLock(
+		filepath.Join(repoRoot, devritespaths.DevritesRootName), slug, fn)
+}
+
 // Create path-disjoint-gates, creates 2-10 worktrees from base, and writes a running lease.
-func Create(opts CreateOpts) (*Lease, error) {
+func Create(opts CreateOpts) (lease *Lease, err error) {
+	err = withLeaseLock(opts.Root, opts.Slug, func() error {
+		lease, err = createLocked(opts)
+		return err
+	})
+	return lease, err
+}
+
+func createLocked(opts CreateOpts) (*Lease, error) {
 	if err := validateSlug(opts.Slug); err != nil {
 		return nil, err
 	}
+	// Store absolute worktree paths in the lease so cleanup's destructive ops
+	// never depend on the caller's working directory.
+	if abs, err := filepath.Abs(opts.Root); err == nil {
+		opts.Root = abs
+	}
 	if err := validateBatchID(opts.BatchID); err != nil {
 		return nil, err
 	}
@@ -123,11 +151,20 @@ func Create(opts CreateOpts) (*Lease, error) {
 		cleanupPartial()
 		return nil, err
 	}
+	ensureScratchExcluded(opts.Root)
 	return lease, nil
 }
 
 // RecordGreen marks a slice green with its transfer commit.
-func RecordGreen(repoRoot, slug, sliceID, commit string) (*Lease, error) {
+func RecordGreen(repoRoot, slug, sliceID, commit string) (lease *Lease, err error) {
+	err = withLeaseLock(repoRoot, slug, func() error {
+		lease, err = recordGreenLocked(repoRoot, slug, sliceID, commit)
+		return err
+	})
+	return lease, err
+}
+
+func recordGreenLocked(repoRoot, slug, sliceID, commit string) (*Lease, error) {
 	leasePath, err := LeasePath(repoRoot, slug)
 	if err != nil {
 		return nil, err
@@ -136,8 +173,8 @@ func RecordGreen(repoRoot, slug, sliceID, commit string) (*Lease, error) {
 	if err != nil {
 		return nil, err
 	}
-	if lease.Status != StatusRunning {
-		return nil, fmt.Errorf("record-green requires status=running (have %s)", lease.Status)
+	if lease.Status != StatusRunning && lease.Status != StatusIntegrateFailed {
+		return nil, fmt.Errorf("record-green requires status=running|integrate-failed (have %s)", lease.Status)
 	}
 	sha, err := revParse(repoRoot, commit)
 	if err != nil {
@@ -161,8 +198,17 @@ func RecordGreen(repoRoot, slug, sliceID, commit string) (*Lease, error) {
 	return lease, nil
 }
 
-// Abort marks the lease aborted and restores control tip to base_sha.
-func Abort(repoRoot, slug string, force bool) (*Lease, error) {
+// Abort marks the lease aborted. Control is never rewound: a moved HEAD is
+// external work, so divergence is reported, not reset.
+func Abort(repoRoot, slug string) (lease *Lease, err error) {
+	err = withLeaseLock(repoRoot, slug, func() error {
+		lease, err = abortLocked(repoRoot, slug)
+		return err
+	})
+	return lease, err
+}
+
+func abortLocked(repoRoot, slug string) (*Lease, error) {
 	leasePath, err := LeasePath(repoRoot, slug)
 	if err != nil {
 		return nil, err
@@ -171,12 +217,15 @@ func Abort(repoRoot, slug string, force bool) (*Lease, error) {
 	if err != nil {
 		return nil, err
 	}
+	if lease.Status == StatusComplete {
+		return nil, fmt.Errorf("abort refused: lease already complete — the batch landed in control; nothing to discard")
+	}
 	base, err := revParse(repoRoot, lease.BaseSHA)
 	if err != nil {
 		return nil, err
 	}
-	if err := ensureControlAtBase(repoRoot, base, force); err != nil {
-		return nil, err
+	if err := ensureControlAtBase(repoRoot, base); err != nil {
+		warnf("control not at base: %v", err)
 	}
 	lease.Status = StatusAborted
 	if err := WriteLease(leasePath, lease); err != nil {
@@ -185,7 +234,10 @@ func Abort(repoRoot, slug string, force bool) (*Lease, error) {
 	return lease, nil
 }
 
-func ensureControlAtBase(repoRoot, base string, force bool) error {
+// ensureControlAtBase verifies the control tip still equals the batch base.
+// It never mutates: a moved or dirty control belongs to the user, so callers
+// only learn about divergence — they must not repair it by rewinding.
+func ensureControlAtBase(repoRoot, base string) error {
 	head, err := headSHA(repoRoot)
 	if err != nil {
 		return err
@@ -193,24 +245,7 @@ func ensureControlAtBase(repoRoot, base string, force bool) error {
 	if head == base {
 		return nil
 	}
-	dirty, err := porcelainDirty(repoRoot)
-	if err != nil {
-		return err
-	}
-	if dirty && !force {
-		return fmt.Errorf("control tree dirty; refuse reset to base (pass --force to override)")
-	}
-	if err := resetHard(repoRoot, base); err != nil {
-		return err
-	}
-	head, err = headSHA(repoRoot)
-	if err != nil {
-		return err
-	}
-	if head != base {
-		return fmt.Errorf("failed to leave control at base %s", base)
-	}
-	return nil
+	return fmt.Errorf("control head %s moved past base %s; refusing to rewind external commits", head, base)
 }
 
 // IntegrateOpts configures staging integrate.
@@ -218,11 +253,18 @@ type IntegrateOpts struct {
 	Root           string
 	Slug           string
 	ApplyToControl bool
-	Force          bool
 }
 
 // Integrate all-or-nothing applies sibling transfer commits onto a staging branch.
 func Integrate(opts IntegrateOpts) (tip string, lease *Lease, err error) {
+	err = withLeaseLock(opts.Root, opts.Slug, func() error {
+		tip, lease, err = integrateLocked(opts)
+		return err
+	})
+	return tip, lease, err
+}
+
+func integrateLocked(opts IntegrateOpts) (tip string, lease *Lease, err error) {
 	leasePath, err := LeasePath(opts.Root, opts.Slug)
 	if err != nil {
 		return "", nil, err
@@ -231,17 +273,36 @@ func Integrate(opts IntegrateOpts) (tip string, lease *Lease, err error) {
 	if err != nil {
 		return "", nil, err
 	}
-	if lease.Status != StatusRunning {
-		return "", nil, fmt.Errorf("integrate requires status=running (have %s)", lease.Status)
+	if lease.Status != StatusRunning && lease.Status != StatusIntegrateFailed {
+		return "", nil, fmt.Errorf("integrate requires status=running|integrate-failed (have %s)", lease.Status)
 	}
 	base, err := revParse(opts.Root, lease.BaseSHA)
 	if err != nil {
 		return "", nil, err
 	}
-	if err := ensureControlAtBase(opts.Root, base, opts.Force); err != nil {
+	if err := ensureControlAtBase(opts.Root, base); err != nil {
 		return "", nil, err
 	}
 
+	union := make([]string, 0, len(lease.Slices))
+	ids := make([]string, 0, len(lease.Slices))
+	for _, sl := range lease.Slices {
+		union = append(union, sl.Paths...)
+		ids = append(ids, sl.ID)
+	}
+	if opts.ApplyToControl {
+		// FF into control must not silently absorb or clobber the user's
+		// uncommitted work on slice paths. Non-overlapping dirty files are
+		// left alone — they are not part of the batch.
+		dirty, err := porcelainDirtyPaths(opts.Root, union)
+		if err != nil {
+			return "", nil, err
+		}
+		if dirty {
+			return "", nil, fmt.Errorf("control has uncommitted changes on slice paths; commit or stash them before integrate --apply-to-control")
+		}
+	}
+
 	for _, sl := range lease.Slices {
 		if sl.WrightStatus != WrightGreen {
 			return "", nil, fmt.Errorf("slice %s is not green", sl.ID)
@@ -284,8 +345,8 @@ func Integrate(opts IntegrateOpts) (tip string, lease *Lease, err error) {
 					leasePath, StatusRunning, retryErr)
 			}
 		}
-		if err := ensureControlAtBase(opts.Root, base, true); err != nil {
-			warnf("control reset to base: %v", err)
+		if err := ensureControlAtBase(opts.Root, base); err != nil {
+			warnf("control diverged from base during integrate: %v", err)
 		}
 		return "", lease, reason
 	}
@@ -318,29 +379,16 @@ func Integrate(opts IntegrateOpts) (tip string, lease *Lease, err error) {
 			}
 		}
 
-		stageHead, err := headSHA(stageWT)
-		if err != nil {
-			return fail(err)
-		}
-		canFF, err := isAncestor(opts.Root, stageHead, tc)
-		if err != nil {
-			return fail(err)
-		}
-		if canFF {
-			if err := mergeFFOnly(stageWT, tc); err != nil {
-				return fail(fmt.Errorf("ff-only integrate failed for slice %s: %w", sl.ID, err))
-			}
-		} else {
-			if err := cherryPickRange(stageWT, base, tc); err != nil {
-				cherryPickAbort(stageWT)
-				return fail(fmt.Errorf("cherry-pick replay failed for slice %s: %w", sl.ID, err))
-			}
+		if err := cherryPickNoCommit(stageWT, base, tc); err != nil {
+			cherryPickAbort(stageWT)
+			return fail(fmt.Errorf("squash apply failed for slice %s: %w", sl.ID, err))
 		}
 	}
 
-	tip, err = headSHA(stageWT)
-	if err != nil {
-		return fail(err)
+	msg := fmt.Sprintf("WIP(%s): parallel batch %s (%s)\n\n[devrites-context]\nslices: %s\nbase: %s",
+		opts.Slug, lease.BatchID, strings.Join(ids, ", "), strings.Join(ids, ", "), base)
+	if tip, err = stagePathsAndCommit(stageWT, msg, union); err != nil {
+		return fail(fmt.Errorf("squash commit: %w", err))
 	}
 	if err := removeWorktree(opts.Root, stageWT); err != nil {
 		if _, statErr := os.Stat(stageWT); statErr == nil {
@@ -359,14 +407,16 @@ func Integrate(opts IntegrateOpts) (tip string, lease *Lease, err error) {
 		if err := mergeFFOnly(opts.Root, tip); err != nil {
 			return fail(fmt.Errorf("control fast-forward to integrate tip failed: %w", err))
 		}
-	} else if err := ensureControlAtBase(opts.Root, base, opts.Force); err != nil {
+		lease.Status = StatusComplete
+		if err := WriteLease(leasePath, lease); err != nil {
+			return "", nil, err
+		}
+	} else if err := ensureControlAtBase(opts.Root, base); err != nil {
 		return fail(err)
 	}
-
-	lease.Status = StatusComplete
-	if err := WriteLease(leasePath, lease); err != nil {
-		return "", nil, err
-	}
+	// Without --apply-to-control the lease stays running: the squash commit
+	// lives only on the integrate branch, and a running lease keeps a
+	// non-forced cleanup from discarding the refs that hold it.
 	return tip, lease, nil
 }
 
@@ -382,24 +432,105 @@ func pathListsEqual(a, b []string) bool {
 	return true
 }
 
-// Cleanup removes worker worktrees/branches and clears the lease (complete or --force).
-func Cleanup(repoRoot, slug string, force bool) error {
-	leasePath, err := LeasePath(repoRoot, slug)
+// Salvage records one worker branch kept by a non-complete cleanup: the
+// rejected work stays reachable for mining instead of being discarded.
+// Worktree is set when salvage itself failed and the worktree was left on
+// disk as the only remaining copy.
+type Salvage struct {
+	SliceID  string `json:"slice_id"`
+	Branch   string `json:"branch"`
+	Commit   string `json:"commit"`
+	Worktree string `json:"worktree,omitempty"`
+}
+
+// ownedWorktreePath reports whether the lease's worktree path is exactly the
+// deterministic <scratch>/<batch>/<slice> location Create wrote. Anything else
+// is not ours: cleanup must neither salvage into it nor delete it.
+func ownedWorktreePath(repoRoot, batchID string, sl LeaseSlice) bool {
+	got, err1 := filepath.Abs(sl.WorktreePath)
+	want, err2 := filepath.Abs(WorkerWorktreePath(repoRoot, batchID, sl.ID))
+	return err1 == nil && err2 == nil && got == want
+}
+
+// salvageSlice commits any uncommitted allowlisted changes in a worker
+// worktree onto its slice branch so forced cleanup loses no wright work.
+func salvageSlice(sl LeaseSlice, batchID string) (string, error) {
+	if sl.WorktreePath == "" || len(sl.Paths) == 0 {
+		return "", nil
+	}
+	if _, err := os.Stat(sl.WorktreePath); err != nil {
+		// Worktree already gone; committed work stays reachable on the branch.
+		return "", nil
+	}
+	dirty, err := porcelainDirtyPaths(sl.WorktreePath, sl.Paths)
 	if err != nil {
+		return "", err
+	}
+	if !dirty {
+		return "", nil
+	}
+	return stagePathsAndCommit(sl.WorktreePath,
+		fmt.Sprintf("devrites: salvage %s WIP (%s)", sl.ID, batchID), sl.Paths)
+}
+
+// Cleanup removes worker worktrees and clears the lease. A complete batch
+// deletes the merged slice branches; a forced non-complete cleanup first
+// salvages each sibling — uncommitted allowlisted changes become a commit on
+// the slice branch — and keeps every branch whose tip moved past base as the
+// durable rejected-work ref.
+func Cleanup(repoRoot, slug string, force bool) (salvaged []Salvage, err error) {
+	err = withLeaseLock(repoRoot, slug, func() error {
+		salvaged, err = cleanupLocked(repoRoot, slug, force)
 		return err
+	})
+	return salvaged, err
+}
+
+func cleanupLocked(repoRoot, slug string, force bool) ([]Salvage, error) {
+	leasePath, err := LeasePath(repoRoot, slug)
+	if err != nil {
+		return nil, err
 	}
 	if _, err := os.Stat(leasePath); os.IsNotExist(err) {
-		return nil
+		return nil, nil
 	}
 	lease, err := ReadLease(leasePath)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	if lease.Status != StatusComplete && !force {
-		return fmt.Errorf("cleanup refuses status=%s without --force (abort keeps workers for diagnosis)", lease.Status)
+		return nil, fmt.Errorf("cleanup refuses status=%s without --force (non-complete cleanup salvages work onto slice branches)", lease.Status)
 	}
+	complete := lease.Status == StatusComplete
+	base := ""
+	if !complete {
+		if b, err := revParse(repoRoot, lease.BaseSHA); err != nil {
+			warnf("base sha %s: %v", lease.BaseSHA, err)
+		} else {
+			base = b
+		}
+	}
+	var salvaged []Salvage
+	keptWorktrees := map[string]bool{}
 	for _, sl := range lease.Slices {
-		if sl.WorktreePath != "" {
+		salvageFailed := false
+		// Only the deterministic scratch layout is ours to touch. A forged or
+		// stale lease worktree_path is never salvaged into (git ops would run
+		// in an arbitrary repo) and never removed (RemoveAll would delete an
+		// arbitrary directory).
+		owned := sl.WorktreePath != "" && ownedWorktreePath(repoRoot, lease.BatchID, sl)
+		if sl.WorktreePath != "" && !owned {
+			warnf("slice %s worktree_path %q is outside %s; leaving it untouched",
+				sl.ID, sl.WorktreePath, ScratchRoot(repoRoot))
+		}
+		if !complete && owned {
+			if _, err := salvage(sl, lease.BatchID); err != nil {
+				warnf("salvage %s: %v", sl.ID, err)
+				salvageFailed = true
+				keptWorktrees[sl.ID] = true
+			}
+		}
+		if owned && !salvageFailed {
 			if err := removeWorktree(repoRoot, sl.WorktreePath); err != nil {
 				warnf("worktree cleanup %s: %v", sl.WorktreePath, err)
 			}
@@ -407,23 +538,76 @@ func Cleanup(repoRoot, slug string, force bool) error {
 				warnf("worktree dir cleanup %s: %v", sl.WorktreePath, err)
 			}
 		}
+		if sl.Branch != "" && !ownedBranchName(slug, lease.BatchID, sl.Branch) {
+			warnf("slice %s branch %q is outside devrites/parallel/%s/%s/; leaving it untouched",
+				sl.ID, sl.Branch, slug, lease.BatchID)
+			if salvageFailed {
+				salvaged = append(salvaged, Salvage{SliceID: sl.ID, Worktree: sl.WorktreePath})
+			}
+			continue
+		}
 		if sl.Branch != "" {
-			if err := deleteBranch(repoRoot, sl.Branch); err != nil {
+			keep := salvageFailed
+			tip := ""
+			if !complete {
+				if t, err := revParse(repoRoot, sl.Branch); err != nil {
+					warnf("branch tip %s: %v", sl.Branch, err)
+					keep = true // fail toward preservation
+				} else {
+					tip = t
+					keep = keep || base == "" || tip != base
+				}
+			}
+			if keep {
+				s := Salvage{SliceID: sl.ID, Branch: sl.Branch, Commit: tip}
+				if salvageFailed {
+					s.Worktree = sl.WorktreePath
+				}
+				salvaged = append(salvaged, s)
+			} else if err := deleteBranch(repoRoot, sl.Branch); err != nil {
 				warnf("branch cleanup %s: %v", sl.Branch, err)
 			}
+		} else if salvageFailed {
+			salvaged = append(salvaged, Salvage{SliceID: sl.ID, Worktree: sl.WorktreePath})
 		}
 	}
 	ibranch := IntegrateBranchName(slug, lease.BatchID)
-	if err := deleteBranch(repoRoot, ibranch); err != nil {
-		warnf("integrate branch cleanup %s: %v", ibranch, err)
+	if itip, err := revParse(repoRoot, ibranch); err == nil {
+		keep := true
+		if head, herr := headSHA(repoRoot); herr == nil {
+			if anc, aerr := isAncestor(repoRoot, itip, head); aerr == nil && anc {
+				keep = false
+			}
+		}
+		if keep {
+			salvaged = append(salvaged, Salvage{SliceID: "integrate", Branch: ibranch, Commit: itip})
+		} else if err := deleteBranch(repoRoot, ibranch); err != nil {
+			warnf("integrate branch cleanup %s: %v", ibranch, err)
+		}
 	}
-	if err := os.RemoveAll(filepath.Join(ScratchRoot(repoRoot), lease.BatchID)); err != nil {
-		warnf("scratch cleanup: %v", err)
+	batchDir := filepath.Join(ScratchRoot(repoRoot), lease.BatchID)
+	if len(keptWorktrees) == 0 {
+		if err := os.RemoveAll(batchDir); err != nil {
+			warnf("scratch cleanup: %v", err)
+		}
+	} else if entries, err := os.ReadDir(batchDir); err == nil {
+		// A failed salvage keeps its worktree as the only copy — remove the
+		// batch dir's other entries but never a kept slice dir.
+		for _, e := range entries {
+			if keptWorktrees[e.Name()] {
+				continue
+			}
+			if err := os.RemoveAll(filepath.Join(batchDir, e.Name())); err != nil {
+				warnf("scratch entry cleanup %s: %v", e.Name(), err)
+			}
+		}
+	} else {
+		warnf("scratch cleanup %s: %v", batchDir, err)
 	}
 	if _, err := git(repoRoot, "worktree", "prune"); err != nil {
 		warnf("worktree prune: %v", err)
 	}
-	return ClearLease(leasePath)
+	return salvaged, ClearLease(leasePath)
 }
 
 // StatusReport returns a human-readable lease status.