devrites 5.14.4 → 6.0.0

package/engine/internal/notes/notes_test.go

@@ -1,10 +1,13 @@
 package notes
 
 import (
+	"bytes"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
+
+	"github.com/devrites/devrites/internal/gate"
 )
 
 func writeFile(t *testing.T, root, rel, body string) {
@@ -71,6 +74,40 @@ func TestParseNotesMasksFencedBlocks(t *testing.T) {
 	}
 }
 
+func TestParseNotesMasksListIndentedFences(t *testing.T) {
+	text := "# Notes\n\n- how to write a note\n\n    ```\n    - NOTE-999: fenced fake\n      SUBJECT: x.go\n      QUOTE: y\n    ```\n\n" + sampleNotes
+	doc := ParseNotes(text)
+	if len(doc.Errors) != 0 || len(doc.Notes) != 2 {
+		t.Fatalf("notes=%d errors=%v", len(doc.Notes), doc.Errors)
+	}
+}
+
+func TestOpenBlockingQuestionAfterIndentedCodeStaysVisible(t *testing.T) {
+	for name, text := range map[string]string{
+		"top-level indented code":         "# Q\n\n    - example\n    ```\n\n## Q-001\nstatus: open\ngate: blocking\n",
+		"list indented code":              "- a\n\n      - not a list\n       ```\n\n## Q-001\nstatus: open\ngate: blocking\n",
+		"unclosed list fence":             "- a\n\n    ```\n\n## Q-001\nstatus: open\ngate: blocking\n",
+		"unclosed ordered fence":          "1. step\n\n     ```\n## Q-001\nstatus: open\ngate: blocking\n",
+		"unclosed nested fence":           "- a\n  - b\n\n      ```\n\n## Q-001\nstatus: open\ngate: blocking\n",
+		"thematic break":                  "* * *\n\n    ```\n## Q-001\nstatus: open\ngate: blocking\n",
+		"ordered cannot interrupt":        "text\n2. foo\n\n    ```\n## Q-001\nstatus: open\ngate: blocking\n",
+		"list-indented fence ends":        "- a\n\n    ```\n    ## Q-002\n    ```\n\n## Q-001\nstatus: open\ngate: blocking\n",
+		"empty dash then list fence":      "-\n\n    ```\n\n## Q-001\nstatus: open\ngate: blocking\n",
+		"setext dash then list fence":     "Intro\n-\n\n    ```\n\n## Q-001\nstatus: open\ngate: blocking\n",
+		"empty star then list fence":      "Some text\n*\n\n    ```\n\n## Q-001\nstatus: open\ngate: blocking\n",
+		"empty dash then table":           "-\n\n    ```\n  | ID | Status | Gate |\n  |---|---|---|\n  | Q-001 | open | blocking |\n",
+		"unclosed list fence to EOF":      "- a\n\n    ```\n    text\n\n## Q-001\nstatus: open\ngate: blocking\n",
+		"list fence dedents before close": "- a\n\n    ```\n    text\nback\n    ```\n## Q-001\nstatus: open\ngate: blocking\n",
+		"tab-indented closer then table":  "- a\n\n    ```\n\t```\n  | ID | Status | Gate |\n  |---|---|---|\n  | Q-001 | open | blocking |\n    ```\n",
+	} {
+		t.Run(name, func(t *testing.T) {
+			if got := gate.OpenBlockingQuestionGates([]byte(text)); len(got) != 1 || got[0] != "blocking" {
+				t.Fatalf("OpenBlockingQuestionGates = %v, want [blocking]", got)
+			}
+		})
+	}
+}
+
 func TestGradeExactMovedStaleAmbiguousLost(t *testing.T) {
 	project := t.TempDir()
 	writeFile(t, project, "src/pool.go", "package src\n\nconst maxConns = 4\n")
@@ -171,3 +208,338 @@ func TestRepairSubjectRewritesOnlyThatLine(t *testing.T) {
 		t.Fatal("repair touched the other note")
 	}
 }
+
+// anchorQuote is the verbatim quote every subject in the cases below still
+// carries, so a decline to read the file is the only reason a note is not exact.
+const anchorQuote = "maxConns = 4"
+
+// writeOversizedQuoteSubject grows path past the read cap while keeping the
+// anchor quote.
+func writeOversizedQuoteSubject(t *testing.T, path string) {
+	t.Helper()
+	filler := strings.Repeat("padding line past the read cap\n", maxSearchFileBytes/28+1)
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(path, []byte("package src\n\nconst "+anchorQuote+"\n"+filler), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	info, err := os.Stat(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if info.Size() <= int64(maxSearchFileBytes) {
+		t.Fatalf("oversized subject is %d bytes, not past the %d-byte read cap", info.Size(), maxSearchFileBytes)
+	}
+}
+
+// unreadableSubjects are the ways a subject this pass will not read as text can
+// still contain the quote. Each degrades an existing, quote-bearing subject.
+var unreadableSubjects = map[string]func(t *testing.T, path string){
+	"permission denied": func(t *testing.T, path string) {
+		if err := os.Chmod(path, 0o000); err != nil {
+			t.Fatal(err)
+		}
+		t.Cleanup(func() { _ = os.Chmod(path, 0o644) })
+		if _, err := os.ReadFile(path); err == nil {
+			t.Skip("chmod 000 does not deny reads for this process; the permission decline is not exercised here")
+		}
+	},
+	"nul prefixed": func(t *testing.T, path string) {
+		if err := os.WriteFile(path, []byte("\x00package src\n\nconst "+anchorQuote+"\n"), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	},
+}
+
+// subjectProject builds a project whose quote-bearing subject this pass will
+// decline, next to one unrelated file carrying the same quote. That second
+// file is the trap: only a wrongly-run relocation scan would name it.
+func subjectProject(t *testing.T, degrade func(t *testing.T, path string)) string {
+	t.Helper()
+	project := t.TempDir()
+	writeFile(t, project, "src/pool.go", "package src\n\nconst "+anchorQuote+"\n")
+	writeFile(t, project, "other/unrelated.txt", "unrelated prose repeating "+anchorQuote+" verbatim\n")
+	degrade(t, filepath.Join(project, "src", "pool.go"))
+	return project
+}
+
+func anchorNotes() string {
+	return Render("", "\n", "NOTE-001", "pool cap rationale", "src/pool.go", anchorQuote, "")
+}
+
+// A subject this pass declines to read is not evidence that the quote moved.
+// The assertion is the negative, not the silence: no moved grade, no
+// relocation target, and repairMoved returns the artifact byte-identical.
+func TestUnreadableSubjectIsNotRelocatedOrRewritten(t *testing.T) {
+	for name, degrade := range unreadableSubjects {
+		t.Run(name, func(t *testing.T) {
+			project := subjectProject(t, degrade)
+
+			notesText := anchorNotes()
+			doc := ParseNotes(notesText)
+			if len(doc.Errors) != 0 {
+				t.Fatalf("parse: %v", doc.Errors)
+			}
+			graded := GradeAll(project, doc)
+			if graded[0].Grade == GradeExact {
+				t.Fatalf("declined subject graded exact without being read: %+v", graded[0])
+			}
+			if graded[0].Grade == GradeMoved {
+				t.Fatalf("declined subject graded moved -> %q", graded[0].MovedTo)
+			}
+			if graded[0].MovedTo != "" {
+				t.Fatalf("declined subject carries a relocation target: %q", graded[0].MovedTo)
+			}
+
+			repaired, repairedText := repairMoved(notesText, doc, graded)
+			if repaired != 0 {
+				t.Fatalf("repairMoved rewrote %d note(s) it did not positively determine moved", repaired)
+			}
+			if repairedText != notesText {
+				t.Fatalf("repairMoved changed the artifact:\n got %q\nwant %q", repairedText, notesText)
+			}
+		})
+	}
+}
+
+// cmdAdd re-verifies the quote against the subject before writing. It must
+// refuse a subject the pass will not read as text, so a note can never be
+// created against one.
+func TestAddRefusesSubjectItCannotReadAsText(t *testing.T) {
+	project := t.TempDir()
+	root := filepath.Join(project, ".devrites")
+	work := filepath.Join(root, "work", "feat")
+	if err := os.MkdirAll(work, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	writeFile(t, project, "src/pool.go", "\x00package src\n\nconst "+anchorQuote+"\n")
+
+	t.Setenv("DEVRITES_WORKSPACE", "")
+	var stdout, stderr bytes.Buffer
+	if code := cmdAdd(root, []string{"feat", "src/pool.go", anchorQuote, "pool cap rationale"},
+		&stdout, &stderr); code != ExitBlocked {
+		t.Fatalf("note add exit = %d, want %d for an unreadable subject:\n%s%s",
+			code, ExitBlocked, stdout.String(), stderr.String())
+	}
+	if _, err := os.Stat(filepath.Join(work, NotesFile)); err == nil {
+		t.Fatal("note add wrote notes.md for a subject it could not read")
+	}
+}
+
+// A subject past the read cap that still carries the quote anchors exactly. The
+// cap bounds relocation scanning, not whether a note can be anchored at all.
+func TestOversizedSubjectWithQuoteGradesExactAndAnchors(t *testing.T) {
+	project := t.TempDir()
+	writeFile(t, project, "other/unrelated.txt", "unrelated prose repeating "+anchorQuote+" verbatim\n")
+	writeOversizedQuoteSubject(t, filepath.Join(project, "src", "pool.go"))
+
+	doc := ParseNotes(anchorNotes())
+	graded := GradeAll(project, doc)
+	if graded[0].Grade != GradeExact || graded[0].MovedTo != "" || graded[0].Detail != "" {
+		t.Fatalf("oversized quote-bearing subject = %+v, want exact", graded[0])
+	}
+
+	root := filepath.Join(project, ".devrites")
+	work := filepath.Join(root, "work", "feat")
+	if err := os.MkdirAll(work, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("DEVRITES_WORKSPACE", "")
+	var stdout, stderr bytes.Buffer
+	if code := cmdAdd(root, []string{"feat", "src/pool.go", anchorQuote, "pool cap rationale"},
+		&stdout, &stderr); code != ExitOK {
+		t.Fatalf("note add exit = %d, want %d for an oversized subject carrying the quote:\n%s%s",
+			code, ExitOK, stdout.String(), stderr.String())
+	}
+	stdout.Reset()
+	if code := cmdCheck(root, []string{"feat"}, &stdout, &stderr); code != ExitOK {
+		t.Fatalf("note check exit = %d, want %d:\n%s%s", code, ExitOK, stdout.String(), stderr.String())
+	}
+}
+
+// An oversized subject that has lost the quote is read to the end and graded on
+// what it holds, so the relocation scan may then name the single other home.
+func TestOversizedSubjectWithoutQuoteIsRegradedFromItsContent(t *testing.T) {
+	project := t.TempDir()
+	writeFile(t, project, "other/unrelated.txt", "unrelated prose repeating "+anchorQuote+" verbatim\n")
+	subject := filepath.Join(project, "src", "pool.go")
+	writeOversizedQuoteSubject(t, subject)
+	if err := os.WriteFile(subject, bytes.ReplaceAll(mustRead(t, subject), []byte(anchorQuote), []byte("maxConns = 8")), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	graded := GradeAll(project, ParseNotes(anchorNotes()))
+	if graded[0].Grade != GradeMoved || graded[0].MovedTo != "other/unrelated.txt" {
+		t.Fatalf("oversized subject without the quote = %+v, want moved -> other/unrelated.txt", graded[0])
+	}
+}
+
+func mustRead(t *testing.T, path string) []byte {
+	t.Helper()
+	b, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return b
+}
+
+// The quote is matched after whitespace normalization, so it must be found when
+// the raw whitespace around and inside it straddles any internal read boundary.
+func TestSubjectQuoteFoundAcrossReadBoundaries(t *testing.T) {
+	for _, pad := range []int{0, 1, 4095, 4096, 4097, 32764, 32766, 32767, 32768, 32769, 32770, 65520, 65530, 65533, 65534, 65535, 65536, 65537, 131070, 131072} {
+		for _, sp := range []string{" ", "\n\t ", "\u00a0\u2003"} {
+			project := t.TempDir()
+			body := strings.Repeat("\n", pad) + "const maxConns" + sp + "=" + sp + "4" + sp + "tail\n"
+			writeFile(t, project, "src/pool.go", body)
+			if g := GradeNote(project, &Note{Subject: "src/pool.go", Quote: anchorQuote}); g.Grade != GradeExact {
+				t.Fatalf("pad %d spacing %q graded %+v, want exact", pad, sp, g)
+			}
+			writeFile(t, project, "src/pool.go", strings.Repeat("\n", pad)+"const maxConns"+sp+"=")
+			if g := GradeNote(project, &Note{Subject: "src/pool.go", Quote: anchorQuote}); g.Grade == GradeExact {
+				t.Fatalf("pad %d spacing %q graded exact for a truncated quote", pad, sp)
+			}
+		}
+	}
+}
+
+// The rewrite is the harm, so the CLI oracle is a byte comparison of notes.md
+// across `note check --repair`, not a line-count or a "no false positive".
+func TestCheckRepairLeavesNotesByteIdenticalWhenSubjectUnreadable(t *testing.T) {
+	for name, degrade := range unreadableSubjects {
+		t.Run(name, func(t *testing.T) {
+			project := t.TempDir()
+			subjectPath := filepath.Join(project, "src", "pool.go")
+			writeFile(t, project, "src/pool.go", "package src\n\nconst "+anchorQuote+"\n")
+			writeFile(t, project, "other/unrelated.txt", "unrelated prose repeating "+anchorQuote+" verbatim\n")
+			root := filepath.Join(project, ".devrites")
+			work := filepath.Join(root, "work", "feat")
+			if err := os.MkdirAll(work, 0o755); err != nil {
+				t.Fatal(err)
+			}
+			notesPath := filepath.Join(work, NotesFile)
+			if err := os.WriteFile(notesPath, []byte(anchorNotes()), 0o644); err != nil {
+				t.Fatal(err)
+			}
+			degrade(t, subjectPath)
+
+			before, err := os.ReadFile(notesPath)
+			if err != nil {
+				t.Fatal(err)
+			}
+			t.Setenv("DEVRITES_WORKSPACE", "")
+			var stdout, stderr bytes.Buffer
+			code := cmdCheck(root, []string{"feat", "--repair"}, &stdout, &stderr)
+
+			// Check the rewrite first: re-pointing the anchor, even to a
+			// correct location, is itself the harm for a subject that was not read.
+			after, err := os.ReadFile(notesPath)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !bytes.Equal(before, after) {
+				t.Fatalf("notes.md was rewritten in place:\nbefore %q\nafter  %q", before, after)
+			}
+			if code == ExitOK {
+				t.Fatalf("note check --repair exited 0 for a subject it never read:\n%s%s", stdout.String(), stderr.String())
+			}
+			if !strings.Contains(stdout.String(), "result: drifted") {
+				t.Fatalf("want a drifted result, got:\n%s", stdout.String())
+			}
+			for _, line := range strings.Split(stdout.String(), "\n") {
+				if strings.HasPrefix(line, "note: NOTE-001 moved") {
+					t.Fatalf("reported a relocation the scan was never entitled to run: %q\n%s", line, stdout.String())
+				}
+			}
+		})
+	}
+}
+
+// A relocation scan that could not enter part of the tree has not seen every
+// candidate, so a single hit among the readable files is not a unique location.
+func TestPartialScanDoesNotGradeSingleHitMoved(t *testing.T) {
+	project := t.TempDir()
+	writeFile(t, project, "visible/a.txt", "prose repeating "+anchorQuote+" verbatim\n")
+	writeFile(t, project, "hidden/b.txt", "prose repeating "+anchorQuote+" verbatim\n")
+	hidden := filepath.Join(project, "hidden")
+	if err := os.Chmod(hidden, 0o000); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = os.Chmod(hidden, 0o755) })
+	if _, err := os.ReadDir(hidden); err == nil {
+		t.Skip("chmod 000 does not deny directory reads for this process")
+	}
+	root := filepath.Join(project, ".devrites")
+	work := filepath.Join(root, "work", "feat")
+	if err := os.MkdirAll(work, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	notesPath := filepath.Join(work, NotesFile)
+	if err := os.WriteFile(notesPath, []byte(anchorNotes()), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	doc := ParseNotes(anchorNotes())
+	graded := GradeAll(project, doc)
+	if graded[0].Grade != GradeAmbiguous {
+		t.Fatalf("grade = %s (moved to %q), want %s for a partial scan", graded[0].Grade, graded[0].MovedTo, GradeAmbiguous)
+	}
+
+	before, err := os.ReadFile(notesPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("DEVRITES_WORKSPACE", "")
+	var stdout, stderr bytes.Buffer
+	cmdCheck(root, []string{"feat", "--repair"}, &stdout, &stderr)
+	after, err := os.ReadFile(notesPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(before, after) {
+		t.Fatalf("notes.md was rewritten from a partial scan:\nbefore %q\nafter  %q", before, after)
+	}
+}
+
+// Several notes whose subjects are gone share one repository walk: each
+// candidate file is read once however many notes are looking for a quote, and
+// every note keeps the grade and location it would get on its own.
+func TestGradeAllReadsEachCandidateOnce(t *testing.T) {
+	project := t.TempDir()
+	writeFile(t, project, "a.txt", "alpha quote one lives here\n")
+	writeFile(t, project, "b.txt", "beta quote two lives here\n")
+	writeFile(t, project, "c.txt", "beta quote two lives here too\n")
+	writeFile(t, project, "d.txt", "nothing relevant\n")
+	doc := ParseNotes(`- NOTE-001: one
+  SUBJECT: gone/one.txt
+  QUOTE: alpha quote one
+- NOTE-002: two
+  SUBJECT: gone/two.txt
+  QUOTE: beta quote two
+- NOTE-003: three
+  SUBJECT: gone/three.txt
+  QUOTE: absent from everywhere
+`)
+	reads := map[string]int{}
+	orig := readCandidate
+	readCandidate = func(path string) fileRead {
+		reads[filepath.Base(path)]++
+		return orig(path)
+	}
+	t.Cleanup(func() { readCandidate = orig })
+
+	graded := GradeAll(project, doc)
+	for name, n := range reads {
+		if n > 1 {
+			t.Errorf("%s read %d times, want at most once", name, n)
+		}
+	}
+	if graded[0].Grade != GradeMoved || graded[0].MovedTo != "a.txt" {
+		t.Errorf("note 1: %+v", graded[0])
+	}
+	if graded[1].Grade != GradeAmbiguous {
+		t.Errorf("note 2: %+v", graded[1])
+	}
+	if graded[2].Grade != GradeLost {
+		t.Errorf("note 3: %+v", graded[2])
+	}
+}

package/engine/internal/overhaul/admit/packet.go

@@ -0,0 +1,89 @@
+package admit
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/devrites/devrites/internal/overhaul/ovio"
+	"github.com/devrites/devrites/internal/overhaul/packet"
+	"github.com/devrites/devrites/internal/overhaul/records"
+)
+
+// packetAdmit checks a dispatch packet file against the run area and the
+// dispatch.json attempt it names. The packet must sit, unlinked, at
+// RUN/packets/<name>.json and pass packet.Validate for that run, which reads
+// a relative receipt_path from the working directory, as packet write does.
+func packetAdmit(run, path string, stdout io.Writer) (int, error) {
+	cur, err := os.ReadFile(join(run, "CURRENT"))
+	if err != nil {
+		return 0, err
+	}
+	gen := workingGeneration(run, strings.TrimSpace(string(cur)))
+	rj, err := ovio.LoadObject(join(gen, "run.json"))
+	if err != nil {
+		return 0, err
+	}
+	disp, err := ovio.LoadObject(join(gen, "dispatch.json"))
+	if err != nil {
+		return 0, err
+	}
+	p, err := ovio.LoadObject(path)
+	if err != nil {
+		return 0, err
+	}
+	eq := func(a, b any) bool { return records.Repr(a) == records.Repr(b) }
+
+	// Resolved once, so every check below sees the same run location whatever
+	// the working directory. receipt_path is judged exactly as packet write
+	// judges it: unchanged, by packet.Validate.
+	absRun, err := filepath.Abs(run)
+	if err != nil {
+		return 0, err
+	}
+	canon := filepath.Join(absRun, "receipts", packet.Name(p)+".json")
+	why := packet.Validate(absRun, p)
+
+	if st, err := os.Lstat(path); err != nil || st.Mode()&os.ModeSymlink != 0 {
+		why = append(why, "packet file must not be a symlink")
+	}
+	if filepath.Base(path) != packet.Name(p)+".json" {
+		why = append(why, "file name does not match run_id, task_id, attempt_id and role")
+	}
+	if realpath(filepath.Dir(path)) != filepath.Join(realpath(run), "packets") {
+		why = append(why, "packet is not under the run's packets directory")
+	}
+	if !eq(p["run_id"], rj["run_id"]) {
+		why = append(why, "run_id differs from the run")
+	}
+
+	var x map[string]any
+	for _, it := range ovio.List(disp["attempts"]) {
+		if a := ovio.Obj(it); eq(a["task_id"], p["task_id"]) && eq(a["attempt_id"], p["attempt_id"]) {
+			x = a
+			break
+		}
+	}
+	if x == nil {
+		why = append(why, "no such dispatched attempt")
+		return result(stdout, why, "ADMISSIBLE"), nil
+	}
+	if s := ovio.Str(x["status"]); s == "admitted" || s == "rejected" || s == "cancelled" {
+		verdict(stdout, "DUPLICATE_OR_LATE", "attempt_status", x["status"])
+		return 3, nil
+	}
+	if !eq(p["role"], x["role"]) {
+		why = append(why, "role differs from the dispatched attempt")
+	}
+	if ovio.Truthy(x["snapshot"]) && !eq(p["snapshot"], x["snapshot"]) {
+		why = append(why, "snapshot differs from the dispatched attempt")
+	}
+	if r := ovio.Str(x["receipt"]); r != "" && filepath.Clean(join(absRun, r)) != canon {
+		why = append(why, "receipt_path differs from the dispatched attempt's receipt")
+	}
+	if pp := ovio.Str(x["packet"]); pp != "" && filepath.Clean(join(absRun, pp)) != filepath.Join(absRun, "packets", packet.Name(p)+".json") {
+		why = append(why, "packet path differs from the dispatched attempt's packet")
+	}
+	return result(stdout, why, "ADMISSIBLE"), nil
+}