devrites 6.1.1 → 6.1.3

package/engine/internal/lib/taskgraph.go

@@ -2,8 +2,11 @@ package lib
 
 import (
 	"fmt"
+	"path"
+	"sort"
 	"strings"
 
+	"github.com/devrites/devrites/internal/parallel"
 	"github.com/devrites/devrites/internal/state"
 )
 
@@ -27,6 +30,127 @@ func CheckTaskGraph(root, slug string) (TaskGraphResult, error) {
 	return ParseTaskGraph(raw), nil
 }
 
+// hubAdvisoryMin is how many slices must write one path before it is flagged:
+// slices sharing a write path never share a parallel build round.
+const hubAdvisoryMin = 3
+
+// hubAdvisoryShown caps printed hub lines; the metric still counts every hub.
+const hubAdvisoryShown = 5
+
+// sliceWritePaths returns each slice's write set (Writer allowlist, else Files
+// likely touched) and whether it records a `Serial rationale:`.
+func sliceWritePaths(tasksMarkdown []byte, graph TaskGraphResult) (map[string][]string, map[string]bool) {
+	paths := make(map[string][]string, len(graph.Slices))
+	serial := make(map[string]bool)
+	for _, s := range graph.Slices {
+		block, _ := state.ExtractTaskSlice(tasksMarkdown, s.ID)
+		fields := sliceFields(block)
+		allow, _ := fieldValue(fields, "writer allowlist")
+		paths[s.ID] = pathTokens(allow)
+		if len(paths[s.ID]) == 0 {
+			touched, _ := fieldValue(fields, "files likely touched")
+			paths[s.ID] = pathTokens(touched)
+		}
+		if v, _ := fieldValue(fields, "serial rationale"); v != "" {
+			serial[s.ID] = true
+		}
+	}
+	return paths, serial
+}
+
+// pathTokens keeps the list entries that can be paths; prose and Markdown links
+// (anything with whitespace) are not write paths.
+func pathTokens(raw string) []string {
+	var out []string
+	for _, p := range parsePathList(raw) {
+		if !strings.ContainsAny(p, " \t") {
+			out = append(out, p)
+		}
+	}
+	return out
+}
+
+// scheduleRounds replays /rite-build batch selection at the maximum cap: each
+// round runs the plan-order path-disjoint subset of dependency-ready slices.
+// It returns 0 when a slice has no write paths or parallel select rejects them.
+func scheduleRounds(graph TaskGraphResult, paths map[string][]string, root string) int {
+	done := make(map[string]bool, len(graph.Slices))
+	rounds := 0
+	for len(done) < len(graph.Slices) {
+		var ready []parallel.SlicePaths
+		for _, s := range graph.Slices {
+			if done[s.ID] || !allDone(s.Dependencies, done) {
+				continue
+			}
+			if len(paths[s.ID]) == 0 {
+				return 0
+			}
+			ready = append(ready, parallel.SlicePaths{ID: s.ID, Paths: paths[s.ID]})
+		}
+		if len(ready) == 0 {
+			return 0
+		}
+		picked, err := parallel.SelectGreedy(parallel.MaxParallelSlices, ready, root)
+		if err != nil || len(picked) == 0 {
+			return 0
+		}
+		for _, p := range picked {
+			done[p.ID] = true
+		}
+		rounds++
+	}
+	return rounds
+}
+
+func allDone(ids []string, done map[string]bool) bool {
+	for _, id := range ids {
+		if !done[id] {
+			return false
+		}
+	}
+	return true
+}
+
+// hubAdvisories names each path written by hubAdvisoryMin+ slices that carry
+// no `Serial rationale:`, most-shared first.
+func hubAdvisories(graph TaskGraphResult, paths map[string][]string, serial map[string]bool) []string {
+	writers := map[string][]string{}
+	shown := map[string]string{} // first spelling seen, for display
+	for _, s := range graph.Slices {
+		if serial[s.ID] {
+			continue
+		}
+		seen := map[string]bool{}
+		for _, p := range paths[s.ID] {
+			key := strings.ToLower(path.Clean(strings.ReplaceAll(p, `\`, "/")))
+			if !seen[key] {
+				seen[key] = true
+				writers[key] = append(writers[key], s.ID)
+				if _, ok := shown[key]; !ok {
+					shown[key] = p
+				}
+			}
+		}
+	}
+	hubs := make([]string, 0, len(writers))
+	for p, ids := range writers {
+		if len(ids) >= hubAdvisoryMin {
+			hubs = append(hubs, p)
+		}
+	}
+	sort.Slice(hubs, func(i, j int) bool {
+		if len(writers[hubs[i]]) != len(writers[hubs[j]]) {
+			return len(writers[hubs[i]]) > len(writers[hubs[j]])
+		}
+		return hubs[i] < hubs[j]
+	})
+	lines := make([]string, len(hubs))
+	for i, p := range hubs {
+		lines[i] = fmt.Sprintf("ADVISORY hub: %s is written by %d slices (%s), so they build one at a time; move the shared edit into one seam slice they depend on, or record `Serial rationale:`", shown[p], len(writers[p]), strings.Join(writers[p], ", "))
+	}
+	return lines
+}
+
 // edgeAdvisories names each dependency edge that does not say what crosses it:
 // no `(via <interface>)`, or a via the dependency's Consumes / Produces lacks.
 // An edge the dependent reads nothing across only serializes work.

package/engine/internal/parallel/cli.go

@@ -74,18 +74,23 @@ func runPathDisjoint(args []string, stdin io.Reader, stdout, stderr io.Writer) i
 
 func cmdSelect(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
 	root := ""
+	slug := ""
 	jsonPath := "-"
 	capN := 0
 	sawCap := false
 	for i := 0; i < len(args); i++ {
 		switch args[i] {
-		case "--root":
+		case "--root", "--slug":
 			if i+1 >= len(args) {
 				fmt.Fprintln(stderr, usageSelect)
 				return ExitUsage
 			}
 			i++
-			root = args[i]
+			if args[i-1] == "--root" {
+				root = args[i]
+			} else {
+				slug = args[i]
+			}
 		case "--cap":
 			if i+1 >= len(args) {
 				fmt.Fprintln(stderr, usageSelect)
@@ -124,7 +129,33 @@ func cmdSelect(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
 		fmt.Fprintf(stderr, "parallel select: %v\n", err)
 		return ExitBlocked
 	}
-	selected, err := SelectGreedy(capN, slices, root)
+	var seed []SlicePaths
+	running := -1
+	if slug != "" {
+		lease, err := runningLease(root, slug)
+		if err != nil {
+			fmt.Fprintf(stderr, "parallel select: %v\n", err)
+			return ExitBlocked
+		}
+		if lease != nil {
+			// The lease's active slices keep their slots and paths; nothing it
+			// already names (active or landed) is picked again.
+			seed = activeSlices(lease)
+			running = len(seed)
+			named := make(map[string]bool, len(lease.Slices))
+			for _, sl := range lease.Slices {
+				named[sl.ID] = true
+			}
+			var fresh []SlicePaths
+			for _, s := range slices {
+				if !named[s.ID] {
+					fresh = append(fresh, s)
+				}
+			}
+			slices = fresh
+		}
+	}
+	selected, err := SelectGreedySeeded(capN, seed, slices, root)
 	if err != nil {
 		fmt.Fprintf(stderr, "parallel select: %v\n", err)
 		if strings.Contains(err.Error(), "cap must be") {
@@ -132,10 +163,30 @@ func cmdSelect(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
 		}
 		return ExitBlocked
 	}
-	fmt.Fprintln(stdout, formatSelect(selected))
+	fmt.Fprintln(stdout, formatSelect(selected, running))
 	return ExitOK
 }
 
+// runningLease returns slug's lease when it is still in flight
+// (running|integrate-failed), or nil when there is none to seed from.
+func runningLease(root, slug string) (*Lease, error) {
+	path, err := LeasePath(root, slug)
+	if err != nil {
+		return nil, err
+	}
+	lease, err := ReadLease(path)
+	if os.IsNotExist(err) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	if lease.Status != StatusRunning && lease.Status != StatusIntegrateFailed {
+		return nil, nil
+	}
+	return lease, nil
+}
+
 func runParallel(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
 	if len(args) == 0 {
 		fmt.Fprintln(stderr, parallelUsage())
@@ -145,6 +196,8 @@ func runParallel(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
 	switch sub {
 	case "create":
 		return cmdCreate(rest, stdin, stdout, stderr)
+	case "add":
+		return cmdAdd(rest, stdin, stdout, stderr)
 	case "record-green":
 		return cmdRecordGreen(rest, stdout, stderr)
 	case "abort":
@@ -174,16 +227,17 @@ func runParallel(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
 
 const (
 	usageCreate       = "usage: parallel create --root --slug --batch --base --json"
+	usageAdd          = "usage: parallel add --root --slug --json <json-file>|-"
 	usageRecordGreen  = "usage: parallel record-green --root --slug --slice --commit"
 	usageAbort        = "usage: parallel abort --root --slug"
-	usageIntegrate    = "usage: parallel integrate --root --slug [--apply-to-control]"
+	usageIntegrate    = "usage: parallel integrate --root --slug [--apply-to-control [--slice <id>]]"
 	usageCleanup      = "usage: parallel cleanup --root --slug [--force]"
 	usageStatus       = "usage: parallel status --root --slug"
 	usageLeaseWrite   = "usage: parallel lease-write --root --slug --json"
 	usageLeaseRead    = "usage: parallel lease-read --root --slug [--field name]"
 	usageLeaseClear   = "usage: parallel lease-clear --root --slug"
 	usagePathDisjoint = "usage: check path-disjoint [--root <dir>] [<json-file>|-]"
-	usageSelect       = "usage: parallel select --cap <1-10> [--root <dir>] [<json-file>|-]"
+	usageSelect       = "usage: parallel select --cap <1-10> [--root <dir>] [--slug <slug>] [<json-file>|-]"
 )
 
 func parallelUsage() string {
@@ -191,15 +245,16 @@ func parallelUsage() string {
 
 Subcommands:
   create          --root --slug --batch --base --json
+  add             --root --slug --json
   record-green    --root --slug --slice --commit
   abort           --root --slug
-  integrate       --root --slug [--apply-to-control]
+  integrate       --root --slug [--apply-to-control [--slice <id>]]
   cleanup         --root --slug [--force]
   status          --root --slug
   lease-write     --root --slug --json
   lease-read      --root --slug [--field name]
   lease-clear     --root --slug
-  select          --cap <1-10> [--root] [<json-file>|-]
+  select          --cap <1-10> [--root] [--slug] [<json-file>|-]
 
 Exit codes: 0 ok, 2 usage, 3 blocked`)
 }
@@ -210,6 +265,8 @@ func CommandUsage(name string) string {
 	switch name {
 	case "create":
 		return usageCreate
+	case "add":
+		return usageAdd
 	case "record-green":
 		return usageRecordGreen
 	case "abort":
@@ -414,6 +471,41 @@ func cmdCreate(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
 	return ExitOK
 }
 
+func cmdAdd(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+	f, code, done := parseFlagsOrHelp(args, stdout, stderr, usageAdd)
+	if done {
+		return code
+	}
+	if code := requireRootSlug(f, stderr); code != ExitOK {
+		return code
+	}
+	if code := rejectUnsupportedFlags(f, stderr, false, false); code != ExitOK {
+		return code
+	}
+	if f.JSON == "" {
+		fmt.Fprintln(stderr, usageAdd)
+		return ExitUsage
+	}
+	data, err := readJSONInput(f.JSON, stdin)
+	if err != nil {
+		fmt.Fprintf(stderr, "parallel add: %v\n", err)
+		return ExitBlocked
+	}
+	slices, err := ParseSlicesJSON(data)
+	if err != nil {
+		fmt.Fprintf(stderr, "parallel add: %v\n", err)
+		return ExitBlocked
+	}
+	lease, err := Add(AddOpts{Root: f.Root, Slug: f.Slug, Slices: slices})
+	if err != nil {
+		fmt.Fprintf(stderr, "parallel add: %v\n", err)
+		return ExitBlocked
+	}
+	fmt.Fprintf(stdout, "added: %d worktrees; batch=%s active=%d base=%s\n",
+		len(slices), lease.BatchID, len(activeSlices(lease)), lease.Slices[len(lease.Slices)-1].BaseSHA)
+	return ExitOK
+}
+
 // rejectUnsupportedFlags reports flags a subcommand parses but must not
 // silently ignore: --force only exists for cleanup, --apply-to-control only
 // for integrate.
@@ -484,16 +576,26 @@ func cmdIntegrate(args []string, stdout, stderr io.Writer) int {
 	if code := rejectUnsupportedFlags(f, stderr, false, true); code != ExitOK {
 		return code
 	}
+	if f.Slice != "" && !f.ApplyToControl {
+		fmt.Fprintln(stderr, "parallel integrate: --slice requires --apply-to-control")
+		fmt.Fprintln(stderr, usageIntegrate)
+		return ExitUsage
+	}
 	tip, lease, err := Integrate(IntegrateOpts{
 		Root:           f.Root,
 		Slug:           f.Slug,
+		Slice:          f.Slice,
 		ApplyToControl: f.ApplyToControl,
 	})
 	if err != nil {
 		fmt.Fprintf(stderr, "parallel integrate: %v\n", err)
 		return ExitBlocked
 	}
-	fmt.Fprintf(stdout, "integrate-ok: tip=%s status=%s apply_to_control=%v\n", tip, lease.Status, f.ApplyToControl)
+	fmt.Fprintf(stdout, "integrate-ok: tip=%s status=%s apply_to_control=%v", tip, lease.Status, f.ApplyToControl)
+	if f.Slice != "" {
+		fmt.Fprintf(stdout, " slice=%s", f.Slice)
+	}
+	fmt.Fprintln(stdout)
 	return ExitOK
 }
 

package/engine/internal/parallel/lease.go

@@ -22,6 +22,9 @@ const (
 	WrightGreen   = "green"
 	WrightRed     = "red"
 	WrightGap     = "gap"
+	// WrightIntegrated marks a slice a per-slice integrate already landed on
+	// control; it no longer holds a parallel slot.
+	WrightIntegrated = "integrated"
 
 	MinParallelSlices = 2
 	MaxParallelSlices = 10
@@ -36,17 +39,22 @@ var (
 )
 
 // Lease is the machine-readable parallel batch lease (control tree only).
+// ControlTip is control HEAD after this lease's most recent per-slice landing;
+// empty means BaseSHA (nothing landed yet).
 type Lease struct {
 	BatchID             string       `json:"batch_id"`
 	CreatedAt           string       `json:"created_at"`
 	BaseSHA             string       `json:"base_sha"`
+	ControlTip          string       `json:"control_tip,omitempty"`
 	N                   int          `json:"n"`
 	Status              string       `json:"status"`
 	ControlPIDOrSession string       `json:"control_pid_or_session"`
 	Slices              []LeaseSlice `json:"slices"`
 }
 
-// LeaseSlice records one worker worktree under the lease.
+// LeaseSlice records one worker worktree under the lease. BaseSHA is the
+// commit its worktree was created from; empty means Lease.BaseSHA (a slice
+// from the original create).
 type LeaseSlice struct {
 	ID             string   `json:"id"`
 	Paths          []string `json:"paths"`
@@ -54,6 +62,35 @@ type LeaseSlice struct {
 	Branch         string   `json:"branch"`
 	WrightStatus   string   `json:"wright_status"`
 	TransferCommit string   `json:"transfer_commit,omitempty"`
+	BaseSHA        string   `json:"base_sha,omitempty"`
+}
+
+// expectedControl is where control must sit: the tip of this lease's latest
+// landing, or the batch base before any landing.
+func expectedControl(lease *Lease) string {
+	if lease.ControlTip != "" {
+		return lease.ControlTip
+	}
+	return lease.BaseSHA
+}
+
+// sliceBase is the commit sl's worktree was created from.
+func sliceBase(lease *Lease, sl LeaseSlice) string {
+	if sl.BaseSHA != "" {
+		return sl.BaseSHA
+	}
+	return lease.BaseSHA
+}
+
+// activeSlices returns the slices that still hold a parallel slot.
+func activeSlices(lease *Lease) []SlicePaths {
+	var active []SlicePaths
+	for _, sl := range lease.Slices {
+		if sl.WrightStatus != WrightIntegrated {
+			active = append(active, SlicePaths{ID: sl.ID, Paths: sl.Paths})
+		}
+	}
+	return active
 }
 
 func LeasePath(repoRoot, slug string) (string, error) {
@@ -132,8 +169,16 @@ func ValidateLease(lease *Lease) error {
 	default:
 		return fmt.Errorf("invalid lease status %q", lease.Status)
 	}
-	if err := checkParallelSliceCount(len(lease.Slices)); err != nil {
-		return err
+	if lease.ControlTip != "" && !shaRE.MatchString(lease.ControlTip) {
+		return fmt.Errorf("invalid control_tip %q", lease.ControlTip)
+	}
+	// A rolling lease grows past the cap as landed slices free their slots;
+	// only the slices still active are bounded.
+	if len(lease.Slices) < MinParallelSlices {
+		return checkParallelSliceCount(len(lease.Slices))
+	}
+	if active := len(activeSlices(lease)); active > MaxParallelSlices {
+		return fmt.Errorf("parallel requires %d-%d slices active at once (got %d)", MinParallelSlices, MaxParallelSlices, active)
 	}
 	if lease.N != len(lease.Slices) {
 		return fmt.Errorf("lease n=%d does not match slices=%d", lease.N, len(lease.Slices))
@@ -151,13 +196,16 @@ func ValidateLease(lease *Lease) error {
 			return err
 		}
 		switch sl.WrightStatus {
-		case "", WrightPending, WrightGreen, WrightRed, WrightGap:
+		case "", WrightPending, WrightGreen, WrightRed, WrightGap, WrightIntegrated:
 		default:
 			return fmt.Errorf("slice %q: invalid wright_status %q", sl.ID, sl.WrightStatus)
 		}
 		if sl.TransferCommit != "" && !shaRE.MatchString(sl.TransferCommit) {
 			return fmt.Errorf("slice %q: invalid transfer_commit %q", sl.ID, sl.TransferCommit)
 		}
+		if sl.BaseSHA != "" && !shaRE.MatchString(sl.BaseSHA) {
+			return fmt.Errorf("slice %q: invalid base_sha %q", sl.ID, sl.BaseSHA)
+		}
 	}
 	return nil
 }
@@ -176,6 +224,9 @@ func EmitLeaseMarkdown(lease *Lease) (string, error) {
 	fmt.Fprintf(&b, "batch_id: %s\n", yamlQuote(lease.BatchID))
 	fmt.Fprintf(&b, "created_at: %s\n", yamlQuote(lease.CreatedAt))
 	fmt.Fprintf(&b, "base_sha: %s\n", yamlQuote(lease.BaseSHA))
+	if lease.ControlTip != "" {
+		fmt.Fprintf(&b, "control_tip: %s\n", yamlQuote(lease.ControlTip))
+	}
 	fmt.Fprintf(&b, "n: %d\n", lease.N)
 	fmt.Fprintf(&b, "status: %s\n", yamlQuote(lease.Status))
 	fmt.Fprintf(&b, "control_pid_or_session: %s\n", yamlQuote(lease.ControlPIDOrSession))
@@ -192,6 +243,9 @@ func EmitLeaseMarkdown(lease *Lease) (string, error) {
 		}
 		fmt.Fprintf(&b, "    worktree_path: %s\n", yamlQuote(sl.WorktreePath))
 		fmt.Fprintf(&b, "    branch: %s\n", yamlQuote(sl.Branch))
+		if sl.BaseSHA != "" {
+			fmt.Fprintf(&b, "    base_sha: %s\n", yamlQuote(sl.BaseSHA))
+		}
 		status := sl.WrightStatus
 		if status == "" {
 			status = WrightPending