devrites 2.6.1 → 3.0.0

package/engine/internal/profile/profile.go

@@ -0,0 +1,270 @@
+package profile
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"sort"
+	"strings"
+	"time"
+)
+
+const schemaVersion = "1"
+
+type FileInfo struct {
+	SHA256 string `json:"sha256"`
+	Bytes  int    `json:"bytes"`
+}
+
+type Profile struct {
+	ProfileSchemaVersion string              `json:"profile_schema_version"`
+	RootSHA              string              `json:"root_sha"`
+	HeadSHA              string              `json:"head_sha"`
+	TopLevel             []string            `json:"top_level"`
+	Manifests            []string            `json:"manifests"`
+	Inputs               map[string]FileInfo `json:"inputs"`
+}
+
+type cacheEnvelope struct {
+	ProfileSchemaVersion string  `json:"profile_schema_version"`
+	RootSHA              string  `json:"root_sha"`
+	HeadSHA              string  `json:"head_sha"`
+	Profile              Profile `json:"profile"`
+}
+
+func Run(args []string, stdout, stderr io.Writer) int {
+	if len(args) != 1 || (args[0] != "get" && args[0] != "refresh") {
+		fmt.Fprintln(stderr, "usage: devrites-engine profile get|refresh")
+		return 2
+	}
+	root, ok := gitRoot()
+	if !ok {
+		fmt.Fprintln(stdout, "NO-CACHE")
+		return 0
+	}
+	rootSHA, headSHA, ok := keys(root)
+	if !ok {
+		fmt.Fprintln(stdout, "NO-CACHE")
+		return 0
+	}
+	path := cachePath(rootSHA, headSHA)
+	switch args[0] {
+	case "get":
+		if cleanProfileInputs(root) {
+			if p, ok := readCache(path, rootSHA, headSHA); ok {
+				fmt.Fprintln(stdout, "HIT")
+				writeJSON(stdout, p)
+				return 0
+			}
+		}
+		fmt.Fprintln(stdout, "MISS")
+		fmt.Fprintln(stdout, path)
+		return 0
+	case "refresh":
+		p := derive(root, rootSHA, headSHA)
+		if err := writeCache(path, p); err != nil {
+			fmt.Fprintf(stderr, "devrites: profile cache write failed: %v\n", err)
+			return 1
+		}
+		fmt.Fprintf(stdout, "WROTE\n%s\n", path)
+		writeJSON(stdout, p)
+		return 0
+	}
+	return 2
+}
+
+func gitRoot() (string, bool) {
+	out, ok := git("", "rev-parse", "--show-toplevel")
+	if !ok || strings.TrimSpace(out) == "" {
+		return "", false
+	}
+	return strings.TrimSpace(out), true
+}
+
+func keys(root string) (string, string, bool) {
+	roots, ok := git(root, "rev-list", "--max-parents=0", "HEAD")
+	if !ok || strings.TrimSpace(roots) == "" {
+		return "", "", false
+	}
+	parts := strings.Fields(roots)
+	sort.Strings(parts)
+	head, ok := git(root, "rev-parse", "HEAD")
+	if !ok || strings.TrimSpace(head) == "" {
+		return "", "", false
+	}
+	return parts[0], strings.TrimSpace(head), true
+}
+
+func cachePath(rootSHA, headSHA string) string {
+	base := os.Getenv("DEVRITES_PROFILE_CACHE")
+	if base == "" {
+		base = "/tmp/compound-engineering/devrites/repo-profile"
+	}
+	return filepath.Join(base, rootSHA, headSHA+".json")
+}
+
+func readCache(path, rootSHA, headSHA string) (Profile, bool) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return Profile{}, false
+	}
+	var env cacheEnvelope
+	if err := json.Unmarshal(raw, &env); err != nil {
+		return Profile{}, false
+	}
+	if env.ProfileSchemaVersion != schemaVersion || env.RootSHA != rootSHA || env.HeadSHA != headSHA || env.Profile.ProfileSchemaVersion != schemaVersion {
+		return Profile{}, false
+	}
+	return env.Profile, true
+}
+
+func writeCache(path string, p Profile) error {
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		return fmt.Errorf("write profile cache %s: %w", path, err)
+	}
+	env := cacheEnvelope{ProfileSchemaVersion: schemaVersion, RootSHA: p.RootSHA, HeadSHA: p.HeadSHA, Profile: p}
+	raw, err := json.MarshalIndent(env, "", "  ")
+	if err != nil {
+		return fmt.Errorf("marshal profile cache: %w", err)
+	}
+	tmp, err := os.CreateTemp(filepath.Dir(path), ".profile-*.tmp")
+	if err != nil {
+		return fmt.Errorf("write profile cache %s: %w", path, err)
+	}
+	name := tmp.Name()
+	if _, err := tmp.Write(append(raw, '\n')); err != nil {
+		_ = tmp.Close()
+		_ = os.Remove(name)
+		return fmt.Errorf("write profile cache %s: %w", path, err)
+	}
+	if err := tmp.Close(); err != nil {
+		_ = os.Remove(name)
+		return fmt.Errorf("write profile cache %s: %w", path, err)
+	}
+	return os.Rename(name, path)
+}
+
+func derive(root, rootSHA, headSHA string) Profile {
+	p := Profile{ProfileSchemaVersion: schemaVersion, RootSHA: rootSHA, HeadSHA: headSHA, Inputs: map[string]FileInfo{}}
+	entries, _ := os.ReadDir(root)
+	for _, e := range entries {
+		name := e.Name()
+		if strings.HasPrefix(name, ".git") || name == "node_modules" {
+			continue
+		}
+		if e.IsDir() {
+			name += "/"
+		}
+		p.TopLevel = append(p.TopLevel, name)
+	}
+	sort.Strings(p.TopLevel)
+	_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
+		if err != nil {
+			return nil
+		}
+		rel, err := filepath.Rel(root, path)
+		if err != nil || rel == "." {
+			return nil
+		}
+		rel = filepath.ToSlash(rel)
+		if d.IsDir() {
+			base := d.Name()
+			if base == ".git" || base == "node_modules" || base == "pack" || base == ".scratch" || base == ".research" {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+		if isManifest(rel) {
+			p.Manifests = append(p.Manifests, rel)
+		}
+		if isProfileInput(rel) {
+			if info, ok := fileInfo(path); ok {
+				p.Inputs[rel] = info
+			}
+		}
+		return nil
+	})
+	sort.Strings(p.Manifests)
+	return p
+}
+
+func fileInfo(path string) (FileInfo, bool) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return FileInfo{}, false
+	}
+	sum := sha256.Sum256(raw)
+	return FileInfo{SHA256: hex.EncodeToString(sum[:]), Bytes: len(raw)}, true
+}
+
+func cleanProfileInputs(root string) bool {
+	out, ok := git(root, "status", "--porcelain", "--untracked-files=all")
+	if !ok {
+		return false
+	}
+	for _, line := range strings.Split(out, "\n") {
+		if strings.TrimSpace(line) == "" {
+			continue
+		}
+		rest := ""
+		if len(line) >= 3 {
+			rest = line[3:]
+		}
+		for _, part := range strings.Split(rest, " -> ") {
+			if isProfileInput(strings.Trim(part, " \t\"")) {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+func isProfileInput(rel string) bool {
+	base := filepath.Base(rel)
+	if isManifest(rel) || matchAny(base, "LICENSE", "LICENSE.md", "LICENSE.txt", "CONTEXT.md", "README.md", "AGENTS.md", "CLAUDE.md", "GEMINI.md", "package.json") {
+		return true
+	}
+	if !strings.Contains(rel, "/") && matchAny(base, "CONTRIBUTING.md", "SECURITY.md", "Makefile") {
+		return true
+	}
+	if strings.HasPrefix(rel, "docs/adr/") || strings.HasPrefix(rel, ".github/workflows/") || strings.HasPrefix(rel, ".devrites/principles.md") || strings.HasPrefix(rel, ".devrites/conventions.md") {
+		return true
+	}
+	return false
+}
+
+func isManifest(rel string) bool {
+	base := filepath.Base(rel)
+	return matchAny(base,
+		"package.json", "package-lock.json", "pnpm-lock.yaml", "yarn.lock", "bun.lock", "go.mod", "go.sum", "Cargo.toml", "Cargo.lock", "pyproject.toml", "requirements.txt", "Gemfile", "Gemfile.lock", "composer.json", "composer.lock", "pom.xml", "build.gradle", "build.gradle.kts", "deno.json", "deno.lock")
+}
+
+func matchAny(s string, vals ...string) bool {
+	return slices.Contains(vals, s)
+}
+
+func git(dir string, args ...string) (string, bool) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, "git", args...)
+	if dir != "" {
+		cmd.Dir = dir
+	}
+	out, err := cmd.Output()
+	if err != nil {
+		return "", false
+	}
+	return string(out), true
+}
+
+func writeJSON(w io.Writer, v any) {
+	raw, _ := json.MarshalIndent(v, "", "  ")
+	fmt.Fprintln(w, string(raw))
+}

package/engine/internal/state/concurrency_test.go

@@ -0,0 +1,184 @@
+package state
+
+// Issue 07: prove the concurrency primitives. These exercise in-process
+// contention (goroutines); the multi-process CLI stress test lives beside the
+// binary (concurrency_cli_test.go).
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+)
+
+// TestAppendLogConcurrentNoInterleaveNoLoss spawns many concurrent appenders,
+// each writing a distinct small record, and asserts every record landed exactly
+// once and intact — no interleaving, no lost writes.
+func TestAppendLogConcurrentNoInterleaveNoLoss(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "events.log")
+	const n = 64
+
+	var wg sync.WaitGroup
+	for i := 0; i < n; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			if err := AppendLog(path, fmt.Sprintf("record-%03d-fixed-width-payload", i)); err != nil {
+				t.Errorf("append %d: %v", i, err)
+			}
+		}(i)
+	}
+	wg.Wait()
+
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	lines := strings.Split(strings.TrimRight(string(raw), "\n"), "\n")
+	if len(lines) != n {
+		t.Fatalf("got %d lines, want %d (lost or torn writes)", len(lines), n)
+	}
+	seen := make(map[string]bool, n)
+	for _, l := range lines {
+		if !strings.HasPrefix(l, "record-") || !strings.HasSuffix(l, "-fixed-width-payload") {
+			t.Fatalf("interleaved/torn record: %q", l)
+		}
+		if seen[l] {
+			t.Fatalf("duplicate record: %q", l)
+		}
+		seen[l] = true
+	}
+}
+
+// TestAtomicWriteReaderNeverSeesPartial has many writers overwrite one path with
+// distinct full-size contents while readers read concurrently. A reader must
+// always observe a complete, self-consistent file — never a half-written or
+// mixed one — which is exactly what temp-file + atomic rename guarantees.
+func TestAtomicWriteReaderNeverSeesPartial(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("Windows does not allow replacing a file while another handle is reading it")
+	}
+	path := filepath.Join(t.TempDir(), "structured.md")
+	const size = 8192
+	if err := AtomicWrite(path, bytes.Repeat([]byte{'A'}, size), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	var writers, readers sync.WaitGroup
+	stop := make(chan struct{})
+
+	// Writers: each writes a homogeneous block of a distinct byte until stopped.
+	for _, b := range []byte("BCDEFGHIJKLMNOP") {
+		writers.Add(1)
+		go func(b byte) {
+			defer writers.Done()
+			for {
+				select {
+				case <-stop:
+					return
+				default:
+					if err := AtomicWrite(path, bytes.Repeat([]byte{b}, size), 0o644); err != nil {
+						t.Errorf("write %c: %v", b, err)
+						return
+					}
+				}
+			}
+		}(b)
+	}
+
+	// Readers: every read must be full-size and homogeneous (one writer's block),
+	// proving no partial or interleaved content is ever visible.
+	for r := 0; r < 4; r++ {
+		readers.Add(1)
+		go func() {
+			defer readers.Done()
+			for i := 0; i < 2000; i++ {
+				data, err := os.ReadFile(path)
+				if err != nil {
+					t.Errorf("read: %v", err)
+					return
+				}
+				if len(data) != size {
+					t.Errorf("partial read: len=%d, want %d", len(data), size)
+					return
+				}
+				first := data[0]
+				for _, c := range data {
+					if c != first {
+						t.Errorf("torn read: mixed bytes %c and %c", first, c)
+						return
+					}
+				}
+			}
+		}()
+	}
+
+	readers.Wait() // readers finish their fixed loops; then stop the writers
+	close(stop)
+	writers.Wait()
+
+	// No temp files left behind after all writes.
+	litter, _ := filepath.Glob(filepath.Join(filepath.Dir(path), ".*.tmp-*"))
+	if len(litter) != 0 {
+		t.Errorf("atomic write left temp litter: %v", litter)
+	}
+}
+
+// TestWithFeatureLockSerializesReadModifyWrite runs a lost-update-prone counter
+// increment concurrently under the per-feature lock. Serialized correctly, the
+// final value is exactly the number of increments.
+func TestWithFeatureLockSerializesReadModifyWrite(t *testing.T) {
+	root := t.TempDir()
+	slug := "counter"
+	if err := os.MkdirAll(featureDir(root, slug), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	counter := filepath.Join(featureDir(root, slug), "count")
+	if err := os.WriteFile(counter, []byte("0"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	const n = 50
+	var wg sync.WaitGroup
+	for i := 0; i < n; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			err := WithFeatureLock(root, slug, func() error {
+				raw, err := os.ReadFile(counter)
+				if err != nil {
+					return err
+				}
+				v, _ := strconv.Atoi(strings.TrimSpace(string(raw)))
+				return os.WriteFile(counter, []byte(strconv.Itoa(v+1)), 0o644)
+			})
+			if err != nil {
+				t.Errorf("locked increment: %v", err)
+			}
+		}()
+	}
+	wg.Wait()
+
+	raw, _ := os.ReadFile(counter)
+	if got := strings.TrimSpace(string(raw)); got != strconv.Itoa(n) {
+		t.Fatalf("counter = %s, want %d — lost updates under contention", got, n)
+	}
+}
+
+// TestAtomicWriteFailsCleanlyOnMissingDir asserts a write whose directory does
+// not exist returns an error rather than a partial file, and leaves no temp
+// litter to find.
+func TestAtomicWriteFailsCleanlyOnMissingDir(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "no-such-dir", "file.md")
+	if err := AtomicWrite(path, []byte("data"), 0o644); err == nil {
+		t.Fatal("expected an error writing into a missing directory")
+	}
+	if _, err := os.Stat(path); !os.IsNotExist(err) {
+		t.Errorf("a failed atomic write left a file behind")
+	}
+}