devrites 6.0.3 → 6.1.0

package/engine/internal/lib/recall.go

@@ -0,0 +1,355 @@
+package lib
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+	"unicode"
+
+	"github.com/devrites/devrites/internal/devritespaths"
+	"github.com/devrites/devrites/internal/markdowntext"
+	"github.com/devrites/devrites/internal/parallel"
+)
+
+// RecallUsage is the help and usage-error text for `recall`.
+const RecallUsage = `usage: devrites-engine recall --paths <p1,p2,...> [--paths <p>]... [--limit n]
+
+Lists archived features (.devrites/archive/<slug>/) whose touched-files.md
+candidate manifest covers a queried project-relative path: exact or as a
+directory prefix, case-folded. Ranked by overlap count, then newest archive
+directory mtime, then slug; --limit caps the features shown (default 10).
+Each feature prints up to 8 pointers (then "+N more"): decisions.md DEC-###
+entries and ## Carry-forward constraints bullets whose Files: overlap (or a
+constraint naming no files), and review.md/seal.md/drift.md lines that mention
+a queried path. Fenced examples are ignored. Pointer files are relative to
+.devrites. Read-only: computed on every call; nothing is written, cached, or
+indexed. An unreadable archive entry prints "unreadable:" and the scan goes on.
+Exit 0 ok (including no archive or no match), 1 invalid archive directory, 2 usage.`
+
+const (
+	recallDefaultLimit = 10
+	recallPointerCap   = 8
+	recallClipRunes    = 120
+)
+
+// recallPointerFiles are scanned in this order; decisions.md is structural,
+// the rest are searched for path mentions.
+var recallPointerFiles = []string{"decisions.md", "review.md", "seal.md", "drift.md"}
+
+var recallDecisionHeading = regexp.MustCompile(`^#{1,6}[ \t]+DEC-[0-9]+`)
+
+type recallFeature struct {
+	slug    string
+	dir     string
+	mtime   time.Time
+	matched []string // queried paths the manifest covers, in query order
+}
+
+// RunRecall answers "what did earlier features decide or find about these
+// paths" by reading the archive on every call (ADR-0014/ADR-0022 forbid an
+// engine index or ledger). It never writes, not even a metric.
+func RunRecall(root string, args []string, stdout, stderr io.Writer) int {
+	queries, limit, ok := parseRecallArgs(args, stderr)
+	if !ok {
+		return 2
+	}
+	archiveDir, err := devritespaths.ArchiveDirChecked(root)
+	if err != nil {
+		fmt.Fprintf(stderr, "recall: invalid archive directory: %v\n", err)
+		return 1
+	}
+	entries, err := os.ReadDir(archiveDir)
+	if errors.Is(err, fs.ErrNotExist) {
+		fmt.Fprintln(stdout, "recall: no archive")
+		return 0
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "recall: read archive: %v\n", err)
+		return 1
+	}
+	var features []recallFeature
+	var unreadable []string
+	for _, entry := range entries {
+		name := entry.Name()
+		// Hidden scratch dirs and stray files are not archived features.
+		if strings.HasPrefix(name, ".") || entry.Type().IsRegular() {
+			continue
+		}
+		label := "archive/" + name
+		dir, err := devritespaths.ExistingArchivedFeatureDirChecked(root, name)
+		if err != nil {
+			unreadable = append(unreadable, fmt.Sprintf("%s (%v)", label, err))
+			continue
+		}
+		raw, err := readBoundedRegularFile(filepath.Join(dir, "touched-files.md"), maxCandidateManifestBytes)
+		if errors.Is(err, fs.ErrNotExist) {
+			continue // never built: touched nothing
+		}
+		var rows []candidateRow
+		if err == nil {
+			rows, err = parseCandidateManifest(raw)
+		}
+		if err != nil {
+			unreadable = append(unreadable, fmt.Sprintf("%s/touched-files.md (%v)", label, err))
+			continue
+		}
+		matched := recallMatched(queries, rows)
+		if len(matched) == 0 {
+			continue
+		}
+		feature := recallFeature{slug: name, dir: dir, matched: matched}
+		if info, err := entry.Info(); err == nil {
+			feature.mtime = info.ModTime()
+		}
+		features = append(features, feature)
+	}
+	sort.SliceStable(features, func(i, j int) bool {
+		a, b := features[i], features[j]
+		if len(a.matched) != len(b.matched) {
+			return len(a.matched) > len(b.matched)
+		}
+		if !a.mtime.Equal(b.mtime) {
+			return a.mtime.After(b.mtime)
+		}
+		return a.slug < b.slug
+	})
+	if len(features) == 0 {
+		fmt.Fprintln(stdout, "recall: no archived feature touched these paths")
+	} else {
+		fmt.Fprintf(stdout, "recall: %d archived feature(s) touched these paths\n", len(features))
+		if len(features) > limit {
+			fmt.Fprintf(stdout, "recall: truncated to %d feature(s) (--limit)\n", limit)
+			features = features[:limit]
+		}
+		for _, f := range features {
+			fmt.Fprintf(stdout, "feature %s overlap=%d paths=%s\n", f.slug, len(f.matched), strings.Join(f.matched, ","))
+			writeRecallPointers(stdout, f.dir, "archive/"+f.slug, queries)
+		}
+	}
+	for _, line := range unreadable {
+		fmt.Fprintf(stdout, "unreadable: %s\n", line)
+	}
+	return 0
+}
+
+func parseRecallArgs(args []string, stderr io.Writer) ([]string, int, bool) {
+	limit := recallDefaultLimit
+	var queries []string
+	seen := map[string]bool{}
+	for i := 0; i < len(args); i++ {
+		switch args[i] {
+		case "--paths":
+			value := argAt(args, i+1)
+			i++
+			if strings.TrimSpace(value) == "" || strings.HasPrefix(value, "-") {
+				fmt.Fprintln(stderr, "recall: --paths requires a comma-separated path list (not a flag)")
+				return nil, 0, false
+			}
+			for _, raw := range strings.Split(value, ",") {
+				if strings.TrimSpace(raw) == "" {
+					continue
+				}
+				path, err := parallel.NormalizePath(raw)
+				if err == nil && strings.HasPrefix(path, "-") {
+					err = fmt.Errorf("path must not look like a flag: %q", raw)
+				}
+				if err != nil {
+					fmt.Fprintf(stderr, "recall: --paths: %v\n", err)
+					return nil, 0, false
+				}
+				if key := strings.ToLower(path); !seen[key] {
+					seen[key] = true
+					queries = append(queries, path)
+				}
+			}
+		case "--limit":
+			n, err := strconv.Atoi(argAt(args, i+1))
+			if err != nil || n < 1 {
+				fmt.Fprintf(stderr, "recall: --limit expects a positive integer, got %q\n", argAt(args, i+1))
+				return nil, 0, false
+			}
+			limit = n
+			i++
+		default:
+			fmt.Fprintf(stderr, "recall: unexpected argument %q\n%s\n", args[i], RecallUsage)
+			return nil, 0, false
+		}
+	}
+	if len(queries) == 0 {
+		fmt.Fprintln(stderr, RecallUsage)
+		return nil, 0, false
+	}
+	return queries, limit, true
+}
+
+// recallMatched returns the queries covered by a manifest row, folding case
+// and matching directory prefixes with the claim-overlap rule.
+func recallMatched(queries []string, rows []candidateRow) []string {
+	var matched []string
+	for _, q := range queries {
+		for _, row := range rows {
+			if claimPathsOverlap(row.path, q) {
+				matched = append(matched, q)
+				break
+			}
+		}
+	}
+	return matched
+}
+
+func writeRecallPointers(w io.Writer, dir, label string, queries []string) {
+	var pointers, unreadable []string
+	for _, name := range recallPointerFiles {
+		raw, err := readBoundedRegularFile(filepath.Join(dir, name), maxCandidateArtifactBytes)
+		if errors.Is(err, fs.ErrNotExist) {
+			continue
+		}
+		if err == nil {
+			raw, err = markdowntext.Structural(raw)
+		}
+		if err != nil {
+			unreadable = append(unreadable, fmt.Sprintf("%s/%s (%v)", label, name, err))
+			continue
+		}
+		lines := strings.Split(string(raw), "\n")
+		var hits []int
+		if name == "decisions.md" {
+			hits = recallDecisionHits(lines, queries)
+		} else {
+			for i, line := range lines {
+				if recallMentions(line, queries) {
+					hits = append(hits, i)
+				}
+			}
+		}
+		for _, i := range hits {
+			pointers = append(pointers, fmt.Sprintf("%s/%s:%d  %s", label, name, i+1, clipRunes(strings.TrimSpace(lines[i]), recallClipRunes)))
+		}
+	}
+	for i, p := range pointers {
+		if i == recallPointerCap {
+			fmt.Fprintf(w, "  +%d more\n", len(pointers)-recallPointerCap)
+			break
+		}
+		fmt.Fprintf(w, "  %s\n", p)
+	}
+	for _, u := range unreadable {
+		fmt.Fprintf(w, "  unreadable: %s\n", u)
+	}
+}
+
+// recallDecisionHits returns the line indexes of DEC-### headings whose
+// Files: field overlaps the queries, and of carry-forward constraint bullets
+// that overlap or name no files. Fenced lines arrive already blanked.
+func recallDecisionHits(lines []string, queries []string) []int {
+	var hits []int
+	dec, decDone, carry := -1, false, false
+	for i, line := range lines {
+		trimmed := strings.TrimSpace(line)
+		rest := strings.TrimLeft(trimmed, "#")
+		if level := len(trimmed) - len(rest); level > 0 && (rest == "" || rest[0] == ' ' || rest[0] == '\t') {
+			dec, decDone = -1, false
+			if recallDecisionHeading.MatchString(trimmed) {
+				dec = i
+			}
+			// As in handoff's dead ends, a ### subheading stays in the section.
+			if level <= 2 {
+				carry = strings.EqualFold(trimmed, "## Carry-forward constraints")
+			}
+			continue
+		}
+		if dec >= 0 {
+			if files, ok := recallFilesField(trimmed); ok && !decDone && recallOverlaps(files, queries) {
+				hits = append(hits, dec)
+				decDone = true
+			}
+			continue
+		}
+		entry, ok := strings.CutPrefix(trimmed, "- ")
+		if !carry || !ok || strings.TrimSpace(entry) == "" {
+			continue
+		}
+		var files []string
+		if at := strings.LastIndex(entry, "Files:"); at >= 0 {
+			files = recallPathList(entry[at+len("Files:"):])
+		}
+		if len(files) == 0 || recallOverlaps(files, queries) {
+			hits = append(hits, i)
+		}
+	}
+	return hits
+}
+
+// recallFilesField parses a `Files:` field line such as "- Files: a, b" or
+// "**Files:** `a` `b`".
+func recallFilesField(trimmed string) ([]string, bool) {
+	s := strings.TrimLeft(trimmed, "-*+ \t")
+	if !strings.HasPrefix(strings.ToLower(s), "files:") {
+		return nil, false
+	}
+	return recallPathList(strings.TrimLeft(s[len("files:"):], "* \t")), true
+}
+
+func recallPathList(s string) []string {
+	switch strings.ToLower(strings.TrimSpace(s)) {
+	case "", "none", "n/a", "-", "—":
+		return nil
+	}
+	var out []string
+	for _, field := range strings.FieldsFunc(s, func(r rune) bool { return r == ',' || r == ';' || unicode.IsSpace(r) }) {
+		field = strings.TrimSuffix(strings.Trim(field, "`'\"()[]"), ".")
+		if path, err := parallel.NormalizePath(field); err == nil {
+			out = append(out, path)
+		}
+	}
+	return out
+}
+
+func recallOverlaps(files, queries []string) bool {
+	for _, f := range files {
+		for _, q := range queries {
+			if claimPathsOverlap(f, q) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// recallMentions reports a case-folded occurrence of a queried path that is
+// not part of a longer path: "src/a.go" matches "`src/a.go:12`" but not
+// "mysrc/a.go", "lib/src/a.go", or "src/a.gox".
+func recallMentions(line string, queries []string) bool {
+	lower := strings.ToLower(line)
+	for _, q := range queries {
+		q = strings.ToLower(q)
+		for from := 0; from < len(lower); {
+			at := strings.Index(lower[from:], q)
+			if at < 0 {
+				break
+			}
+			start, end := from+at, from+at+len(q)
+			before := start == 0 || !recallPathByte(lower[start-1])
+			after := end == len(lower) || lower[end] == '/' || !recallPathByte(lower[end]) ||
+				lower[end] == '.' && (end+1 == len(lower) || !recallPathByte(lower[end+1]))
+			if before && after {
+				return true
+			}
+			from = start + 1
+		}
+	}
+	return false
+}
+
+func recallPathByte(c byte) bool {
+	return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' ||
+		c == '_' || c == '-' || c == '.' || c == '/' || c >= 0x80
+}

package/engine/internal/lib/regression.go

@@ -7,7 +7,9 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"slices"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -49,6 +51,8 @@ var doneSliceStates = map[string]bool{
 var (
 	regressACRe      = regexp.MustCompile(`(?im)^[ \t]*- \[([xX ])\] (AC-\d+)`)
 	regressSliceRe   = regexp.MustCompile(`(?m)^## (SLICE-\d+)\b[^\n]*`)
+	regressBuiltRe   = regexp.MustCompile(`(?m)^[ \t]*[-*] \[[xX]\][^\n]*`)
+	regressSliceIDRe = regexp.MustCompile(`(?i)\b(?:SLICE-|Slice )(\d+)\b`)
 	regressStatusRe  = regexp.MustCompile(`(?im)^Status:[ \t]*([^\s]+)`)
 	regressQHeaderRe = regexp.MustCompile(`(?im)^## (q-\d{4}-\d{2}-\d{2}-\d+)\b`)
 	regressQStatusRe = regexp.MustCompile(`(?im)^status:[ \t]*([^\s]+)`)
@@ -170,6 +174,22 @@ func fingerprintWorkspace(root, slug, workspace string) (progressFingerprint, er
 	if err == nil {
 		fp.DoneSlices = doneSlices(masked)
 	}
+	// The canonical slice grammar has no Status field: Build records the
+	// built list in state.md as checked items naming the slice.
+	masked, err = maskedArtifact(workspace, "state.md")
+	note("state.md", err)
+	if err == nil {
+		for _, item := range regressBuiltRe.FindAllString(masked, -1) {
+			for _, m := range regressSliceIDRe.FindAllStringSubmatch(item, -1) {
+				// `SLICE-001` and `Slice 1` name the same slice.
+				n, _ := strconv.Atoi(m[1])
+				id := fmt.Sprintf("SLICE-%03d", n)
+				if !slices.Contains(fp.DoneSlices, id) {
+					fp.DoneSlices = append(fp.DoneSlices, id)
+				}
+			}
+		}
+	}
 
 	raw, err := os.ReadFile(filepath.Join(workspace, acceptance.GatesFile)) // #nosec G304 -- workspace artifact
 	note(acceptance.GatesFile, err)

package/engine/internal/lib/resolve.go

@@ -6,10 +6,12 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"slices"
 	"strings"
 	"time"
 
 	"github.com/devrites/devrites/internal/fsutil"
+	"github.com/devrites/devrites/internal/markdowntext"
 	"github.com/devrites/devrites/internal/state"
 	"github.com/devrites/devrites/internal/workflow"
 )
@@ -39,16 +41,17 @@ func Resolve(root string, args []string, stdout, stderr io.Writer) int {
 		return fail(stderr, err.Error(), 3)
 	}
 
-	human := argAt(args, 0) == "--human"
-	if human {
-		args = args[1:]
-	}
+	const usage = `Usage: devrites-engine state resolve [--human] <qid> "<answer>"  |  state resolve [--human] --drop <qid> ["<reason>"]  |  state resolve [--human] --batch <file>`
+	human := slices.Contains(args, "--human")
+	args = slices.DeleteFunc(slices.Clone(args), func(arg string) bool { return arg == "--human" })
 	var mode, qid, payload string
+	maxArgs := 2
 	switch first := argAt(args, 0); first {
 	case "--drop":
 		mode = "drop"
 		qid = argAt(args, 1)
 		payload = argAt(args, 2)
+		maxArgs = 3
 		if payload == "" {
 			payload = "dropped"
 		}
@@ -59,7 +62,7 @@ func Resolve(root string, args []string, stdout, stderr io.Writer) int {
 			return fail(stderr, "Batch file not found: "+payload, 5)
 		}
 	case "":
-		return fail(stderr, `Usage: devrites-engine state resolve [--human] <qid> "<answer>"  |  state resolve [--human] --drop <qid> ["<reason>"]  |  state resolve [--human] --batch <file>`, 5)
+		return fail(stderr, usage, 5)
 	default:
 		mode = "answer"
 		qid = first
@@ -68,6 +71,11 @@ func Resolve(root string, args []string, stdout, stderr io.Writer) int {
 			return fail(stderr, "Answer text required for "+qid, 5)
 		}
 	}
+	// An unknown flag or a stray argument must never be recorded as a qid or
+	// an answer.
+	if len(args) > maxArgs || strings.HasPrefix(qid, "-") || mode != "batch" && strings.HasPrefix(payload, "--") {
+		return fail(stderr, usage, 5)
+	}
 
 	code := 0
 	if err := state.WithFeatureLock(root, slug, func() error {
@@ -107,31 +115,49 @@ func resolveMutation(mode, qid, payload, qfile, sfile string, human bool, stdout
 		if err != nil {
 			return fail(stderr, "Batch file not found: "+payload, 5)
 		}
+		// Every line is parsed and applied in memory first, so a bad line
+		// leaves questions.md untouched instead of half-resolved.
+		type batchEntry struct{ qid, status, text string }
+		var entries []batchEntry
 		for _, line := range batchLines(data) {
+			line = strings.TrimSpace(line)
 			if line == "" || strings.HasPrefix(line, "#") {
 				continue
 			}
-			if strings.HasPrefix(line, "--drop") {
-				rest := strings.TrimPrefix(line, "--drop ")
-				bid, reason := splitColon(rest)
-				reason = strings.TrimPrefix(reason, " ")
-				if code := resolveQuestion(qfile, bid, "dropped", reason, human, stderr); code != 0 {
-					return code
-				}
-				if err := clearAwaiting(sfile, bid); err != nil {
-					return fail(stderr, err.Error(), 1)
-				}
-				fmt.Fprintf(stdout, "Dropped:  %s: %s\n", bid, reason)
+			status := "answered"
+			if rest, ok := strings.CutPrefix(line, "--drop "); ok {
+				status, line = "dropped", rest
+			}
+			bid, text, ok := strings.Cut(line, ":")
+			bid = strings.TrimSpace(bid)
+			if !ok || bid == "" || strings.HasPrefix(bid, "-") {
+				return fail(stderr, "Malformed batch line (want `<qid>: <answer>` or `--drop <qid>: <reason>`): "+line, 5)
+			}
+			entries = append(entries, batchEntry{bid, status, strings.TrimPrefix(text, " ")})
+		}
+		// #nosec G304 -- questions.md path inside the feature workspace
+		qdata, err := os.ReadFile(qfile)
+		if err != nil {
+			return fail(stderr, "questions.md missing at "+qfile, 2)
+		}
+		lines := splitLinesNoTrailing(qdata)
+		for _, entry := range entries {
+			var code int
+			if lines, code = applyResolution(lines, entry.qid, entry.status, entry.text, human, stderr); code != 0 {
+				return code
+			}
+		}
+		if err := fsutil.WriteFileAtomic(qfile, joinRecords(lines), 0o644); err != nil {
+			return fail(stderr, err.Error(), 1)
+		}
+		for _, entry := range entries {
+			if err := clearAwaiting(sfile, entry.qid); err != nil {
+				return fail(stderr, err.Error(), 1)
+			}
+			if entry.status == "dropped" {
+				fmt.Fprintf(stdout, "Dropped:  %s: %s\n", entry.qid, entry.text)
 			} else {
-				bid, ans := splitColon(line)
-				ans = strings.TrimPrefix(ans, " ")
-				if code := resolveQuestion(qfile, bid, "answered", ans, human, stderr); code != 0 {
-					return code
-				}
-				if err := clearAwaiting(sfile, bid); err != nil {
-					return fail(stderr, err.Error(), 1)
-				}
-				fmt.Fprintf(stdout, "Resolved: %s\n", bid)
+				fmt.Fprintf(stdout, "Resolved: %s\n", entry.qid)
 			}
 		}
 	}
@@ -171,16 +197,6 @@ func batchLines(data []byte) []string {
 	return lines
 }
 
-// splitColon splits s at the first ':', returning the parts before and after it
-// (both the whole string when there is no ':').
-func splitColon(s string) (before, after string) {
-	before, after, ok := strings.Cut(s, ":")
-	if !ok {
-		return s, s
-	}
-	return before, after
-}
-
 // clockNow is the resolve command's single wall-clock read. DEVRITES_NOW accepts
 // an RFC-3339 timestamp or YYYY-MM-DD date so tests can keep date-derived
 // question IDs stable. See ADR-0006.
@@ -210,16 +226,9 @@ func resolveQuestion(qfile, qid, status, answer string, human bool, stderr io.Wr
 	if err != nil {
 		return fail(stderr, "questions.md missing at "+qfile, 2)
 	}
-	ts := nowUTC()
-	answer = oneLine(answer)
-	target := regexp.MustCompile(`^## ` + regexp.QuoteMeta(qid) + `([[:space:]]|$)`)
-
-	updated, found, notOpen := rewriteQuestionFields(splitLinesNoTrailing(data), target, status, answer, ts, human)
-	if !found {
-		return fail(stderr, "qid not found: "+qid, 3)
-	}
-	if notOpen {
-		return fail(stderr, "qid not open (already answered/dropped): "+qid, 4)
+	updated, code := applyResolution(splitLinesNoTrailing(data), qid, status, answer, human, stderr)
+	if code != 0 {
+		return code
 	}
 	if err := fsutil.WriteFileAtomic(qfile, joinRecords(updated), 0o644); err != nil {
 		return fail(stderr, err.Error(), 1)
@@ -228,10 +237,35 @@ func resolveQuestion(qfile, qid, status, answer string, human bool, stderr io.Wr
 	return 0
 }
 
+// applyResolution rewrites the <qid> block in the questions.md lines in memory.
+// Returns 0, or 3 (qid not found) / 4 (qid not open), writing the message on
+// failure.
+func applyResolution(lines []string, qid, status, answer string, human bool, stderr io.Writer) ([]string, int) {
+	// Locate blocks on the fence-masked text, as the readiness gate does, so a
+	// fenced example is never rewritten and never shadows the real question.
+	var visible []string
+	if len(lines) > 0 {
+		masked, err := markdowntext.Structural([]byte(strings.Join(lines, "\n")))
+		if err != nil {
+			return nil, fail(stderr, "questions.md is not readable Markdown: "+err.Error(), 1)
+		}
+		visible = strings.Split(string(masked), "\n")
+	}
+	target := regexp.MustCompile(`^## ` + regexp.QuoteMeta(qid) + `([[:space:]]|$)`)
+	updated, found, notOpen := rewriteQuestionFields(lines, visible, target, status, oneLine(answer), nowUTC(), human)
+	if !found {
+		return nil, fail(stderr, "qid not found: "+qid, 3)
+	}
+	if notOpen {
+		return nil, fail(stderr, "qid not open (already answered/dropped): "+qid, 4)
+	}
+	return updated, 0
+}
+
 var (
 	qHeaderRe    = regexp.MustCompile(`(?i)^## q-`)
-	statusLineRe = regexp.MustCompile(`^status:`)
-	statusStrip  = regexp.MustCompile(`^status:[[:space:]]*`)
+	statusLineRe = regexp.MustCompile(`(?i)^status:`)
+	statusStrip  = regexp.MustCompile(`(?i)^status:[[:space:]]*`)
 	answeredAtRe = regexp.MustCompile(`^answered_at:`)
 	answerRe     = regexp.MustCompile(`^answer:`)
 	answeredByRe = regexp.MustCompile(`(?i)^[[:space:]]*answered_by:`)
@@ -241,8 +275,9 @@ var (
 // block, sets status and rewrites any answered_at/answer lines already present. It
 // reports whether the block was found and whether it was already not open (an
 // attempt to re-answer). A target block with no status line at all gets a full set
-// of closing fields appended.
-func rewriteQuestionFields(lines []string, target *regexp.Regexp, status, answer, ts string, human bool) (out []string, found, notOpen bool) {
+// of closing fields appended. Structure is matched on visible (lines with
+// fenced blocks masked, same length); unmatched lines are copied from lines.
+func rewriteQuestionFields(lines, visible []string, target *regexp.Regexp, status, answer, ts string, human bool) (out []string, found, notOpen bool) {
 	inQ := false
 	statusSeen, answeredAtSeen, answerSeen, answeredBySeen := false, false, false, false
 	closeBlock := func() {
@@ -262,7 +297,8 @@ func rewriteQuestionFields(lines []string, target *regexp.Regexp, status, answer
 			out = append(out, "answered_by: human")
 		}
 	}
-	for _, line := range lines {
+	for i, original := range lines {
+		line := visible[i]
 		switch {
 		case qHeaderRe.MatchString(line):
 			closeBlock()
@@ -271,11 +307,18 @@ func rewriteQuestionFields(lines []string, target *regexp.Regexp, status, answer
 			if inQ {
 				found = true
 			}
-			out = append(out, line)
+			out = append(out, original)
+		case inQ && hdrSpaceRe.MatchString(line):
+			// Any level-two heading ends the question block, as the gate reads it.
+			closeBlock()
+			inQ = false
+			out = append(out, original)
 		case inQ && statusLineRe.MatchString(line):
 			statusSeen = true
 			cur := statusStrip.ReplaceAllString(line, "")
-			if cur != "open" {
+			// Trim and fold case the way the readiness gate does, so a
+			// question the gate counts as open is always resolvable.
+			if !strings.EqualFold(strings.TrimSpace(cur), "open") {
 				out = append(out, "status: "+cur)
 				inQ = false
 				notOpen = true
@@ -294,7 +337,7 @@ func rewriteQuestionFields(lines []string, target *regexp.Regexp, status, answer
 				out = append(out, "answered_by: human")
 			}
 		default:
-			out = append(out, line)
+			out = append(out, original)
 		}
 	}
 	closeBlock()

package/engine/internal/lib/secretscan.go

@@ -147,7 +147,7 @@ func SecretScan(root string, args []string, stdin io.Reader, stdout, stderr io.W
 		}
 	}
 	if !staged && slug == "" && !readStdin {
-		changed, err := gitDiffNames(project)
+		changed, err := gitChangedAndUntrackedNames(project)
 		if err != nil {
 			fmt.Fprintln(stderr, "secret-scan: cannot inspect changed paths")
 			return 2
@@ -191,6 +191,29 @@ func SecretScan(root string, args []string, stdin io.Reader, stdout, stderr io.W
 	return writeSecretScanResult(findings, stdout, stderr)
 }
 
+// gitChangedAndUntrackedNames lists every path that differs from HEAD (staged
+// or not) plus untracked, non-ignored files. On an unborn branch every tracked
+// path is staged, so the index diff stands in for the HEAD diff. Both lists
+// are relative to project, which scanWorktreePaths joins them onto.
+func gitChangedAndUntrackedNames(project string) ([]string, error) {
+	changed, err := gitDiffNames(project, "--relative", "HEAD", "--")
+	if err != nil {
+		if changed, err = gitDiffNames(project, "--relative", "--cached"); err != nil {
+			return nil, err
+		}
+	}
+	out, err := runGitCommand(project, nil, "ls-files", "--others", "--exclude-standard", "-z")
+	if err != nil {
+		return nil, err
+	}
+	for _, path := range strings.Split(string(out), "\x00") {
+		if path != "" {
+			changed = append(changed, path)
+		}
+	}
+	return changed, nil
+}
+
 func projectDir(root string) string {
 	if filepath.Base(root) == ".devrites" {
 		return filepath.Dir(root)