devrites 5.14.4 → 6.0.0

package/engine/internal/notes/notes.go

@@ -7,12 +7,17 @@
 package notes
 
 import (
+	"bufio"
+	"bytes"
 	"fmt"
+	"io"
 	"io/fs"
 	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
+	"unicode"
+	"unicode/utf8"
 
 	"github.com/devrites/devrites/internal/markdowntext"
 )
@@ -167,7 +172,9 @@ const (
 	// GradeMoved: the quote is absent from the subject and present in exactly
 	// one other repository file; the anchor is repairable.
 	GradeMoved Grade = "moved"
-	// GradeStale: the subject file exists but the quote is gone everywhere.
+	// GradeStale: the subject file exists but the quote is gone everywhere, or
+	// the subject exists and this pass declined to read it as text. Either way
+	// re-anchoring is a human decision.
 	GradeStale Grade = "stale"
 	// GradeAmbiguous: the quote is absent from the subject and present in more
 	// than one other file; re-anchoring needs human judgment.
@@ -177,11 +184,14 @@ const (
 )
 
 // Graded pairs a note with its current grade and, for moved notes, the
-// repository-relative path the quote now lives in.
+// repository-relative path the quote now lives in. Detail explains a grade the
+// grader could not positively determine — currently the read bound that stopped
+// it — so the report never presents an unresolved read as a resolved one.
 type Graded struct {
 	*Note
 	Grade   Grade
 	MovedTo string
+	Detail  string
 }
 
 // normalizeWS collapses whitespace so a quote matches across re-wrapping and
@@ -207,34 +217,179 @@ var skippedDirs = map[string]bool{
 
 const maxSearchFileBytes = 512 << 10
 
-// fileText reads a candidate file as normalized text, skipping oversized and
-// binary files so quotes never match inside archives or images.
-func fileText(path string) (string, bool) {
+// readState is the outcome of reading one file as note text. The three states
+// must stay distinct all the way to the caller: a decline to read is not
+// evidence of absence, and only a proven absence entitles a grader to look
+// for the quote somewhere else.
+type readState int
+
+const (
+	// readOK: the file was read as normalized text.
+	readOK readState = iota
+	// readAbsent: there is no such file.
+	readAbsent
+	// readDeclined: the file is present but this pass did not read it as text.
+	readDeclined
+)
+
+// fileRead carries a read outcome through to its caller. Reason names the bound
+// that refused the read, so a report can say why a subject was not graded
+// instead of implying its quote had been deleted.
+type fileRead struct {
+	text   string
+	state  readState
+	reason string
+	// found is set only by fileHasQuote: the normalized quote occurs in a
+	// file that was read to its end.
+	found bool
+}
+
+// readable reports whether this pass obtained the file's text.
+func (f fileRead) readable() bool { return f.state == readOK }
+
+// statText reports the file's info when it is a regular file; otherwise it
+// returns the absent or declined outcome and false.
+func statText(path string) (os.FileInfo, fileRead, bool) {
 	info, err := os.Stat(path)
-	if err != nil || !info.Mode().IsRegular() || info.Size() > maxSearchFileBytes {
-		return "", false
+	if os.IsNotExist(err) {
+		return nil, fileRead{state: readAbsent}, false
+	}
+	if err != nil {
+		return nil, fileRead{state: readDeclined, reason: "stat failed: " + err.Error()}, false
+	}
+	if !info.Mode().IsRegular() {
+		return nil, fileRead{state: readDeclined, reason: "not a regular file"}, false
+	}
+	return info, fileRead{}, true
+}
+
+// fileText reads a candidate file as normalized text, skipping oversized and
+// binary files so quotes never match inside archives or images. Every decline
+// keeps its own identity: only a missing file reports as absent. A size, mode or
+// permission bound can justify refusing to read a file; none of them is evidence
+// about where its quote went, so none of them may stand in for absence.
+func fileText(path string) fileRead {
+	info, declined, ok := statText(path)
+	if !ok {
+		return declined
+	}
+	if info.Size() > maxSearchFileBytes {
+		return fileRead{state: readDeclined, reason: fmt.Sprintf("%d bytes is over the %d-byte read cap", info.Size(), maxSearchFileBytes)}
 	}
 	// #nosec G304 -- repository scan path under the resolved project root
 	raw, err := os.ReadFile(path)
 	if err != nil {
-		return "", false
+		return fileRead{state: readDeclined, reason: "read failed: " + err.Error()}
 	}
 	head := raw
 	if len(head) > 512 {
 		head = head[:512]
 	}
 	if strings.IndexByte(string(head), 0) >= 0 {
-		return "", false
+		return fileRead{state: readDeclined, reason: "NUL byte in the first 512 bytes"}
+	}
+	return fileRead{text: normalizeWS(string(raw)), state: readOK}
+}
+
+// fileHasQuote reports whether the normalized quote occurs in the file's
+// normalized text, with no size limit. It streams: memory stays bounded by the
+// chunk size plus the quote length, because only the last len(quote)-1
+// normalized bytes carry over between chunks, so a match that straddles a
+// chunk boundary is still found. The size cap exists to bound the repository-wide
+// relocation scan, not to decide whether the subject itself can be anchored.
+func fileHasQuote(path, quote string) fileRead {
+	if _, declined, ok := statText(path); !ok {
+		return declined
+	}
+	// #nosec G304 -- subject path under the resolved project root
+	f, err := os.Open(path)
+	if err != nil {
+		return fileRead{state: readDeclined, reason: "read failed: " + err.Error()}
+	}
+	defer func() { _ = f.Close() }()
+	br := bufio.NewReaderSize(f, 64<<10)
+	head, err := br.Peek(512)
+	if err != nil && err != io.EOF && err != bufio.ErrBufferFull {
+		return fileRead{state: readDeclined, reason: "read failed: " + err.Error()}
+	}
+	if bytes.IndexByte(head, 0) >= 0 {
+		return fileRead{state: readDeclined, reason: "NUL byte in the first 512 bytes"}
+	}
+	q := []byte(quote)
+	var (
+		chunk      = make([]byte, 32<<10)
+		window     []byte // normalized tail carried between chunks, then the new chunk
+		carry      []byte // incomplete trailing UTF-8 sequence
+		wroteWord  bool
+		pendingGap bool
+	)
+	for {
+		n, rerr := br.Read(chunk)
+		if rerr != nil && rerr != io.EOF {
+			return fileRead{state: readDeclined, reason: "read failed: " + rerr.Error()}
+		}
+		eof := rerr == io.EOF
+		raw := append(carry, chunk[:n]...)
+		carry = nil
+		for i := 0; i < len(raw); {
+			if !eof && !utf8.FullRune(raw[i:]) {
+				carry = append([]byte(nil), raw[i:]...)
+				break
+			}
+			r, size := utf8.DecodeRune(raw[i:])
+			if unicode.IsSpace(r) {
+				pendingGap = true
+			} else {
+				if pendingGap && wroteWord {
+					window = append(window, ' ')
+				}
+				pendingGap = false
+				wroteWord = true
+				window = append(window, raw[i:i+size]...)
+			}
+			i += size
+		}
+		if bytes.Contains(window, q) {
+			return fileRead{state: readOK, found: true}
+		}
+		if keep := len(q) - 1; len(window) > keep {
+			window = append(window[:0], window[len(window)-keep:]...)
+		}
+		if eof {
+			return fileRead{state: readOK}
+		}
 	}
-	return normalizeWS(string(raw)), true
 }
 
-// quoteLocations returns every repository-relative file whose normalized text
-// contains the normalized quote, excluding the subject itself.
-func quoteLocations(project, subject, quote string) []string {
-	var found []string
+// relocation is one note whose subject is gone and whose quote the shared
+// repository walk is looking for.
+type relocation struct {
+	note        *Note
+	quote       string
+	subjectRead fileRead
+	found       []string
+	incomplete  bool
+	done        bool
+}
+
+// readCandidate reads one walked file; a variable so tests can count reads.
+var readCandidate = fileText
+
+// locateQuotes walks the project once, reads each file once, and records for
+// every pending note the repository-relative files (excluding its own subject)
+// whose normalized text contains its quote. A note's incomplete flag is set when
+// the walk or a file read failed while it was still scanning, so it may have
+// missed a candidate; size and binary declines are not failures. A note stops
+// scanning once a second hit makes it ambiguous.
+func locateQuotes(project string, pending []*relocation) {
+	active := len(pending)
 	_ = filepath.WalkDir(project, func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
+			for _, r := range pending {
+				if !r.done {
+					r.incomplete = true
+				}
+			}
 			return nil
 		}
 		if d.IsDir() {
@@ -248,52 +403,100 @@ func quoteLocations(project, subject, quote string) []string {
 			return nil
 		}
 		rel = filepath.ToSlash(rel)
-		if rel == subject {
-			return nil
-		}
-		if text, ok := fileText(path); ok && strings.Contains(text, quote) {
-			found = append(found, rel)
-			if len(found) > 1 {
-				return fs.SkipAll // ambiguous already; stop early
+		var read fileRead
+		readDone := false
+		for _, r := range pending {
+			if r.done || rel == r.note.Subject {
+				continue
+			}
+			if !readDone {
+				read, readDone = readCandidate(path), true
+			}
+			if read.state == readDeclined && (strings.HasPrefix(read.reason, "stat failed") || strings.HasPrefix(read.reason, "read failed")) {
+				r.incomplete = true
+			}
+			if read.readable() && strings.Contains(read.text, r.quote) {
+				r.found = append(r.found, rel)
+				if len(r.found) > 1 {
+					r.done = true // ambiguous already; stop scanning for it
+					active--
+				}
 			}
 		}
+		if active == 0 {
+			return fs.SkipAll
+		}
 		return nil
 	})
-	return found
 }
 
-// Grade resolves one note's current anchor state against the project tree.
-// The repo-wide relocation scan runs only when the subject no longer carries
-// the quote, so exact notes cost one file read.
-func GradeNote(project string, note *Note) Graded {
+// gradeSubject grades one note against its subject file alone. The returned
+// relocation is non-nil only when the subject is positively determined gone,
+// so exact notes cost one file read. A subject this pass declines to read is
+// never scanned: the scan is the only evidence that could license re-pointing
+// the anchor, and a declined read is no evidence at all.
+func gradeSubject(project string, note *Note) (Graded, *relocation) {
 	graded := Graded{Note: note}
 	subjectPath := filepath.Join(project, filepath.FromSlash(note.Subject))
-	subjectText, subjectOK := fileText(subjectPath)
 	quote := normalizeWS(note.Quote)
-	if subjectOK && strings.Contains(subjectText, quote) {
+	subjectRead := fileHasQuote(subjectPath, quote)
+	if subjectRead.found {
 		graded.Grade = GradeExact
-		return graded
+		return graded, nil
+	}
+	if subjectRead.state == readDeclined {
+		// Present but unreadable as text. Grade stale so a human re-anchors,
+		// and carry the reason so the grade is not read as a deleted quote.
+		graded.Grade = GradeStale
+		graded.Detail = subjectRead.reason
+		return graded, nil
 	}
-	found := quoteLocations(project, note.Subject, quote)
+	return graded, &relocation{note: note, quote: quote, subjectRead: subjectRead}
+}
+
+// settle turns a finished scan into the note's grade.
+func (r *relocation) settle(graded *Graded) {
 	switch {
-	case len(found) == 1:
+	case len(r.found) == 1 && r.incomplete:
+		// Part of the tree went unread, so this hit may not be the only one.
+		graded.Grade = GradeAmbiguous
+	case len(r.found) == 1:
 		graded.Grade = GradeMoved
-		graded.MovedTo = found[0]
-	case len(found) > 1:
+		graded.MovedTo = r.found[0]
+	case len(r.found) > 1:
 		graded.Grade = GradeAmbiguous
-	case subjectOK:
+	case r.subjectRead.readable():
 		graded.Grade = GradeStale
 	default:
 		graded.Grade = GradeLost
 	}
-	return graded
 }
 
-// GradeAll grades every note in document order.
+// GradeNote resolves one note's current anchor state against the project tree.
+func GradeNote(project string, note *Note) Graded {
+	doc := &Document{Notes: []*Note{note}}
+	return GradeAll(project, doc)[0]
+}
+
+// GradeAll grades every note in document order. Subjects are graded first; the
+// repository-wide relocation scan then runs once for all notes that need it.
 func GradeAll(project string, doc *Document) []Graded {
-	out := make([]Graded, 0, len(doc.Notes))
-	for _, note := range doc.Notes {
-		out = append(out, GradeNote(project, note))
+	out := make([]Graded, len(doc.Notes))
+	var pending []*relocation
+	at := map[*relocation]int{}
+	for i, note := range doc.Notes {
+		g, r := gradeSubject(project, note)
+		out[i] = g
+		if r != nil {
+			pending = append(pending, r)
+			at[r] = i
+		}
+	}
+	if len(pending) > 0 {
+		locateQuotes(project, pending)
+		for _, r := range pending {
+			r.settle(&out[at[r]])
+		}
 	}
 	return out
 }