@aperant/framework 0.6.2

package/src/cli/commands/validate-evidence.mjs

@@ -0,0 +1,537 @@
+/**
+ * commands/validate-evidence.mjs — deterministic Evidence Rule enforcement.
+ *
+ * R6 of /apt:pr-review hardening. Walks a findings directory (markdown
+ * files produced by specialist reviewers, per SCHEMA.md section 3),
+ * parses each `## Finding:` block, and decides whether the claimed
+ * evidence is actually present in the cited file range.
+ *
+ * Rules applied (in order, first-match wins):
+ *
+ *   1. dismissed_evidence_not_literal
+ *      The `evidence` code block must be a literal substring of the
+ *      cited file range (whitespace-normalized). Prose paraphrases and
+ *      search terms get rejected here — no matter how plausible.
+ *
+ *   2. dismissed_multi_file_evidence_missing
+ *      If the description or title references more than one filename
+ *      (regex on file-extension tokens: .ts .tsx .js .jsx .mjs .json
+ *      .md), the finding must also supply an `### Cross-File Evidence`
+ *      block with a literal snippet from each referenced file. Missing
+ *      → dismiss.
+ *
+ * Mutations are written back in place — the findings markdown gets a
+ * new `- **Validation Status**: ${status}` line plus a
+ * `### Validation Note` block. The command always exits 0; the caller
+ * checks the JSON summary on stdout.
+ *
+ * Usage:
+ *   apt-tools validate-evidence <findings-dir> --worktree <path>
+ */
+
+import { existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'
+import { join, resolve } from 'node:path'
+import { parseFlags } from '../util/args.mjs'
+import { err, ok } from '../util/result.mjs'
+
+// Regex of file-extension tokens we use to detect multi-file references
+// inside the structured evidence blocks. Deliberately conservative — the
+// point is to catch findings whose Evidence/Suggested-Fix snippets cite
+// more than one file without supplying a matching `### Cross-File
+// Evidence` entry. Over-detection is NOT OK (bug-4): filenames mentioned
+// in prose fields (title / description / impact / Evidence Context) are
+// just narrative — they must NOT trip the multi-file rule. Only code
+// actually cited as evidence counts. We still strip URLs and fenced code
+// fences from the in-scope sections before matching (see
+// stripNoiseForFileTokens) so that OWASP/RFC links and outer fence
+// markers don't bleed through.
+const FILE_TOKEN_RX = /\b[\w./-]+\.(?:ts|tsx|js|jsx|mjs|json|md)\b/g
+
+// Strip fenced code blocks (```…```) and URL-like substrings (anything
+// after http:// or https:// up to whitespace, closing parens, or
+// markdown-link brackets) BEFORE matching FILE_TOKEN_RX. Inline backtick
+// spans are preserved — those usually wrap real file paths the author
+// wants us to detect (e.g. `auth.ts and session.ts`).
+export function stripNoiseForFileTokens(text) {
+	if (!text) return ''
+	let out = String(text)
+	// 1. Drop fenced code blocks. The opening fence may carry an info
+	//    string (```ts, ```bash, etc.); the closing fence is a bare ```
+	//    on its own (we tolerate trailing whitespace).
+	out = out.replace(/```[^\n]*\n[\s\S]*?```/g, ' ')
+	// 2. Drop URLs. The terminator set covers whitespace, closing parens
+	//    (markdown link form `(https://…)`), brackets, and angle braces.
+	out = out.replace(/https?:\/\/[^\s)\]>]+/g, ' ')
+	return out
+}
+
+/**
+ * Collapse runs of whitespace to a single space and strip leading/
+ * trailing whitespace. We use this on BOTH the claimed evidence and
+ * the target file range so indentation changes / CRLF don't cause a
+ * false mismatch. Keep the normalization identical on both sides.
+ */
+export function normalizeForLiteralMatch(text) {
+	return String(text ?? '')
+		.replace(/\r\n/g, '\n')
+		.replace(/\s+/g, ' ')
+		.trim()
+}
+
+// Annotation marker patterns — line content (after the comment marker is
+// stripped) that indicates the line is a reviewer note rather than source.
+// Examples this matches:
+//   (verified) | (approx) | (approximate) | (elided)
+//   ... N more | ... [N] more identical-pattern setters
+//   rest of … | remaining … | snip | truncated | omitted
+//   lines 106-153 | line 42
+//   bare ellipsis: ... | …
+const ANNOTATION_PHRASE_RX =
+	/\(verified\)|\(approx(?:imate)?\)|\(elided\)|\.\.\.\s*\[?\d+\]?\s*more\b|\brest of\b|\bremaining\b|\bsnip\b|\btruncated\b|\bomitted\b|\blines?\s+\d+(?:\s*-\s*\d+)?\b/i
+
+// A line is "comment-only" if, after trim, it begins with one of the
+// recognized comment markers and contains no code afterwards we'd care
+// about. We deliberately keep the marker set small + language-agnostic;
+// reviewers in this codebase use `//`, `#`, `--`, and `/* … */`.
+const COMMENT_LINE_RX = /^\s*(?:\/\/|#|--|\/\*.*\*\/\s*$|\/\*|\*)/
+
+/**
+ * Strip a single layer of code-fence markers from an evidence string.
+ * The markdown parser already extracts content between the first opening
+ * `` ``` `` fence and the next `` ``` ``, so the "outer" fences are gone
+ * by the time we get here. But reviewers occasionally paste source that
+ * itself contains fence markers (e.g. `` ```bash `` inside the snippet),
+ * leaving stray fence lines behind. This helper removes any line whose
+ * trimmed content is exactly a fence marker — never source code.
+ */
+export function stripCodeFenceLines(text) {
+	if (!text) return ''
+	return String(text)
+		.split(/\r?\n/)
+		.filter((line) => !/^\s*```/.test(line))
+		.join('\n')
+}
+
+/**
+ * Decide whether a single evidence line is a reviewer annotation rather
+ * than source code copied from the cited file.
+ *
+ * A line is treated as an annotation when ALL of the following hold:
+ *   - It is a "comment-only" line (starts with `//`, `#`, `--`, `/*` …).
+ *   - Either it matches one of the well-known annotation phrases (e.g.
+ *     `(verified)`, `... 10 more`, `// rest of`, `// ...`, `lines N-M`),
+ *     OR — fallback rule — its trimmed content does not appear as a
+ *     literal substring of `sourceWindow` (the cited file range plus a
+ *     small ±5-line buffer for reviewer rounding).
+ *
+ * The "appears in sourceWindow" fallback is what lets us accept genuine
+ * comments that happen to be part of the cited source (e.g. a `// TODO`
+ * the reviewer pasted) — those are NOT annotations and should still be
+ * required to match. Only commentary the reviewer added on top is
+ * filtered out.
+ *
+ * Conservative by design: code lines (anything that doesn't start with a
+ * comment marker) are NEVER classified as annotations, so a paraphrased
+ * code line still gets through to the literal-match step and fails.
+ */
+export function isAnnotationLine(line, sourceWindow) {
+	const trimmed = line.trim()
+	if (!trimmed) return false
+	if (!COMMENT_LINE_RX.test(trimmed)) return false
+	// Bare ellipsis-only lines are always annotations: `// ...`, `# ...`,
+	// `-- ...`, or just `...` would have been excluded by COMMENT_LINE_RX
+	// since it requires a marker; but a marker followed only by dots /
+	// whitespace is the canonical truncation cue.
+	if (/^\s*(?:\/\/|#|--|\*)\s*\.{2,}\s*$/.test(trimmed)) return true
+	if (ANNOTATION_PHRASE_RX.test(trimmed)) return true
+	// Fallback: comment doesn't appear in the source window → annotation.
+	const normLine = normalizeForLiteralMatch(trimmed)
+	if (!normLine) return false
+	const normWindow = normalizeForLiteralMatch(sourceWindow)
+	return !normWindow.includes(normLine)
+}
+
+/**
+ * Parse a findings markdown file into an array of findings. Each
+ * finding is a `## Finding: {id} -- {title}` block terminated by the
+ * next `---` (exclusive) or EOF.
+ *
+ * Returns records with:
+ *   { id, title, start, end, raw, fields: {file, line, severity, ...},
+ *     evidence, description, lineRange, crossFileEvidence, hasValidationStatus }
+ *
+ * Mutations re-serialize the raw block, so preserving `start`/`end`
+ * offsets on the original text lets us edit surgically without
+ * rewriting the file's framing.
+ */
+export function parseFindingsMarkdown(markdown) {
+	const findings = []
+	const headerRx = /^## Finding:\s*([A-Z]+-\d+)\s*--\s*(.+)$/gm
+	const matches = [...markdown.matchAll(headerRx)]
+	for (let i = 0; i < matches.length; i++) {
+		const m = matches[i]
+		const start = m.index
+		const nextStart = i + 1 < matches.length ? matches[i + 1].index : markdown.length
+		// Each block ends at the first standalone `---` after its header,
+		// OR at the next header. Be generous — treat either as terminator.
+		const slice = markdown.slice(start, nextStart)
+		const terminator = slice.search(/\n---\s*(?:\n|$)/)
+		const end = terminator >= 0 ? start + terminator : nextStart
+		const body = markdown.slice(start, end)
+		findings.push(parseFindingBlock(body, m[1], m[2], start, end))
+	}
+	return findings
+}
+
+function parseFindingBlock(body, id, title, start, end) {
+	const fields = {}
+	// `- **File**: \`path:line\`` etc.
+	const fieldRx = /^-\s*\*\*([^*]+)\*\*:\s*(.+)$/gm
+	for (const m of body.matchAll(fieldRx)) {
+		const key = m[1].trim().toLowerCase()
+		fields[key] = m[2].trim()
+	}
+
+	// File + line from the "File" field. Accepted shapes:
+	//   `path/to/foo.ts`
+	//   `path/to/foo.ts:42`
+	//   `path/to/foo.ts:42-58`     ← line range
+	// The regex anchors with ^ (without it, lazy [^`:]+? matched the END
+	// of `path:30-32` and captured `30-32` as the file, then validate-evidence
+	// tried to read <worktree>/30-32 and silently dismissed every finding —
+	// FRAMEWORK-BUG-001 surfaced by /apt:pr-review on PR #68 in 2026-04).
+	// `(?:-\d+)?` consumes the trailing range so it doesn't leak into the
+	// file capture.
+	let file = null
+	let line = null
+	if (fields.file) {
+		const fileMatch = /^`?([^`:]+?)(?::(\d+)(?:-\d+)?)?`?\s*$/.exec(fields.file)
+		if (fileMatch) {
+			file = fileMatch[1].trim()
+			line = fileMatch[2] ? Number(fileMatch[2]) : null
+		}
+	}
+
+	// Description block: everything between `### Description` and the next `###`.
+	const description = extractSection(body, 'Description') ?? ''
+
+	// Evidence code block: first triple-backticked block after `### Evidence`.
+	const evidenceSection = extractSection(body, 'Evidence') ?? ''
+	const fenced = /```[^\n]*\n([\s\S]*?)```/.exec(evidenceSection)
+	const evidence = fenced ? fenced[1] : evidenceSection
+
+	// Suggested Fix section (prose + optional fenced snippets). Unlike
+	// Evidence we keep the whole section — snippets AND surrounding
+	// sentences both live here — because file tokens in either form are
+	// legitimate evidence that the fix crosses files.
+	const suggestedFix = extractSection(body, 'Suggested Fix') ?? ''
+
+	// Line range: Evidence Context -> "Lines examined: a-b"
+	const contextSection = extractSection(body, 'Evidence Context') ?? ''
+	const rangeMatch = /Lines? examined[^\d]*(\d+)\s*-\s*(\d+)/i.exec(contextSection)
+	const lineRange = rangeMatch
+		? [Number(rangeMatch[1]), Number(rangeMatch[2])]
+		: line
+			? [line, line]
+			: null
+
+	// Cross-file evidence (optional): `### Cross-File Evidence` block.
+	// Each file gets a `- **File**: path` + fenced snippet.
+	const crossFileSection = extractSection(body, 'Cross-File Evidence') ?? ''
+	const crossFileEvidence = []
+	const crossFileRx = /-\s*\*\*File\*\*:\s*`?([^`\n]+?)`?\s*\n+```[^\n]*\n([\s\S]*?)```/g
+	for (const m of crossFileSection.matchAll(crossFileRx)) {
+		crossFileEvidence.push({ file: m[1].trim(), snippet: m[2] })
+	}
+
+	return {
+		id,
+		title: title.trim(),
+		start,
+		end,
+		raw: body,
+		fields,
+		file,
+		line,
+		lineRange,
+		evidence,
+		description,
+		suggestedFix,
+		crossFileEvidence,
+		hasValidationStatus: /\*\*Validation Status\*\*/i.test(body),
+	}
+}
+
+function extractSection(body, heading) {
+	const rx = new RegExp(`###\\s*${heading}\\s*\\n([\\s\\S]*?)(?=\\n###\\s|$)`, 'i')
+	const m = rx.exec(body)
+	return m ? m[1].trim() : null
+}
+
+/**
+ * Detect that a finding references more than one filename inside its
+ * structured evidence blocks (`### Evidence` + `### Suggested Fix`).
+ * Prose fields (title / description / impact / Evidence Context) are
+ * deliberately NOT scanned — bug-4 proved those routinely mention
+ * filenames in narrative (e.g. "per CLAUDE.md", "see src/foo.ts") that
+ * have nothing to do with whether the evidence itself spans files.
+ *
+ * The Evidence block is usually a pasted code snippet; the Suggested Fix
+ * may be prose, code, or both. We strip fenced-code fences and URL noise
+ * the same way on both sides so outer backticks and OWASP/RFC links
+ * don't bleed through. Returns the sorted unique list of referenced
+ * files — caller decides whether to require cross-file evidence based
+ * on length.
+ */
+export function detectReferencedFiles(finding) {
+	const tokens = new Set()
+	for (const text of [finding.evidence, finding.suggestedFix]) {
+		const cleaned = stripNoiseForFileTokens(text)
+		for (const m of cleaned.matchAll(FILE_TOKEN_RX)) {
+			tokens.add(m[0])
+		}
+	}
+	return [...tokens].sort()
+}
+
+/**
+ * Check a single finding against the literal-substring rule. Returns
+ * a decision object; caller applies it to the markdown.
+ */
+export function checkFinding(finding, worktreeRoot) {
+	// Rule 1: evidence must be a literal substring of the cited file range.
+	if (!finding.file) {
+		return {
+			status: 'dismissed_evidence_not_literal',
+			note: 'Finding has no `file` field — cannot verify evidence literal-match.',
+		}
+	}
+	const abs = resolve(worktreeRoot, finding.file)
+	if (!existsSync(abs)) {
+		return {
+			status: 'dismissed_evidence_not_literal',
+			note: `Cited file not found at ${finding.file} (resolved ${abs}).`,
+		}
+	}
+	const all = readFileSync(abs, 'utf-8').split(/\r?\n/)
+	const [startLine, endLine] = finding.lineRange ?? [1, all.length]
+	// Clamp to file bounds (reviewers sometimes cite past EOF).
+	const safeStart = Math.max(1, Math.min(all.length, startLine))
+	const safeEnd = Math.max(safeStart, Math.min(all.length, endLine))
+	const target = all.slice(safeStart - 1, safeEnd).join('\n')
+	// Buffer ±5 lines around the cited range so the "comment exists in
+	// source?" annotation-detection fallback tolerates small reviewer
+	// rounding errors (e.g. they cite 106-153 but the comment they
+	// quoted is on line 105 or 154). Only the matcher's annotation
+	// detector reads this widened window — the literal-match step still
+	// uses the strict cited range.
+	const bufferedStart = Math.max(1, safeStart - 5)
+	const bufferedEnd = Math.min(all.length, safeEnd + 5)
+	const sourceWindow = all.slice(bufferedStart - 1, bufferedEnd).join('\n')
+
+	const rawEvidence = finding.evidence ?? ''
+	if (!normalizeForLiteralMatch(rawEvidence)) {
+		return {
+			status: 'dismissed_evidence_not_literal',
+			note: 'Finding has empty evidence block.',
+		}
+	}
+
+	// Strip any stray code-fence markers the parser left behind (rare —
+	// the parser already extracts the FIRST fenced block, but reviewers
+	// sometimes nest fences inside the snippet) and split into lines.
+	const evidenceLines = stripCodeFenceLines(rawEvidence).split(/\r?\n/)
+	// Keep only "code-bearing" lines: drop blank lines (whitespace-only
+	// adds no signal) and reviewer annotation comments (per
+	// isAnnotationLine — see helper for the contract).
+	const keptLines = evidenceLines.filter((line) => {
+		if (!line.trim()) return false
+		return !isAnnotationLine(line, sourceWindow)
+	})
+	if (keptLines.length === 0) {
+		// Evidence reduced to nothing matchable after stripping fences +
+		// annotations. Treat as missing evidence — same outcome as an
+		// empty Evidence block.
+		return {
+			status: 'dismissed_evidence_not_literal',
+			note: 'Evidence consists only of fence markers and reviewer annotations — no source content to verify.',
+		}
+	}
+
+	const normTarget = normalizeForLiteralMatch(target)
+	const normWholeEvidence = normalizeForLiteralMatch(keptLines.join('\n'))
+
+	if (normTarget.includes(normWholeEvidence)) {
+		// Fast path: the kept evidence (joined back) is still a literal
+		// contiguous substring of the cited range. This is the common
+		// case — a clean copy-paste with no annotations interleaved.
+	} else {
+		// Slow path: the reviewer interleaved annotations with source
+		// (e.g. `code line A\n// ... 10 more\ncode line B`). Match each
+		// kept line individually against the cited range. Every kept
+		// line must appear; any miss is a paraphrase / fabrication and
+		// the whole finding is rejected.
+		for (const line of keptLines) {
+			const normLine = normalizeForLiteralMatch(line)
+			if (!normLine) continue
+			if (!normTarget.includes(normLine)) {
+				return {
+					status: 'dismissed_evidence_not_literal',
+					note: `Evidence not found as literal substring of ${finding.file} lines ${safeStart}-${safeEnd}: line "${normLine.slice(0, 120)}" missing from cited range.`,
+				}
+			}
+		}
+	}
+
+	// Rule 2: multi-file references require Cross-File Evidence.
+	const referenced = detectReferencedFiles(finding)
+	if (referenced.length > 1) {
+		if (finding.crossFileEvidence.length < referenced.length) {
+			return {
+				status: 'dismissed_multi_file_evidence_missing',
+				note: `Evidence/Suggested Fix reference ${referenced.length} files (${referenced.join(', ')}) but only provides cross-file evidence for ${finding.crossFileEvidence.length}.`,
+			}
+		}
+		// Every referenced file must have a corresponding cross-file
+		// evidence entry whose snippet is a literal substring of that
+		// file. Any miss → dismiss.
+		for (const referencedFile of referenced) {
+			const entry = finding.crossFileEvidence.find((e) => e.file.endsWith(referencedFile))
+			if (!entry) {
+				return {
+					status: 'dismissed_multi_file_evidence_missing',
+					note: `No cross-file evidence entry for referenced file ${referencedFile}.`,
+				}
+			}
+			const crossPath = resolve(worktreeRoot, entry.file)
+			if (!existsSync(crossPath)) {
+				return {
+					status: 'dismissed_multi_file_evidence_missing',
+					note: `Cross-file evidence cites ${entry.file}, which does not exist in the worktree.`,
+				}
+			}
+			const crossTargetNorm = normalizeForLiteralMatch(readFileSync(crossPath, 'utf-8'))
+			const crossEvidenceNorm = normalizeForLiteralMatch(entry.snippet)
+			if (!crossTargetNorm.includes(crossEvidenceNorm)) {
+				return {
+					status: 'dismissed_multi_file_evidence_missing',
+					note: `Cross-file snippet for ${entry.file} is not a literal substring of the file.`,
+				}
+			}
+		}
+	}
+
+	return { status: 'confirmed_valid', note: null }
+}
+
+/**
+ * Rewrite a findings markdown in place, stamping each finding with
+ * `- **Validation Status**: …` + a `### Validation Note` block.
+ */
+function stampFindings(markdown, decisions) {
+	// Work from the end so earlier edits don't shift later offsets.
+	let out = markdown
+	for (let i = decisions.length - 1; i >= 0; i--) {
+		const { finding, decision } = decisions[i]
+		// If the finding already has a Validation Status line, replace
+		// the value; otherwise inject before the first `### Description`
+		// (or at end of body if no such heading).
+		const block = out.slice(finding.start, finding.end)
+		let newBlock
+		if (finding.hasValidationStatus) {
+			newBlock = block.replace(
+				/-\s*\*\*Validation Status\*\*:\s*[^\n]*/i,
+				`- **Validation Status**: ${decision.status}`,
+			)
+			// Replace the note block too, if any.
+			if (decision.note) {
+				if (/###\s*Validation Note/i.test(newBlock)) {
+					newBlock = newBlock.replace(
+						/###\s*Validation Note\s*\n[\s\S]*?(?=\n###|$)/i,
+						`### Validation Note\n${decision.note}\n`,
+					)
+				} else {
+					newBlock = `${newBlock.replace(/\s*$/, '')}\n\n### Validation Note\n${decision.note}\n`
+				}
+			}
+		} else {
+			// Insert "- **Validation Status**: ..." right after the first
+			// `- **...**:` field block.
+			const lastFieldRx = /((?:^-\s*\*\*[^*]+\*\*:.*(?:\n|$))+)/m
+			const match = lastFieldRx.exec(block)
+			const insertion = `- **Validation Status**: ${decision.status}\n`
+			if (match) {
+				const insertAt = match.index + match[0].length
+				newBlock = block.slice(0, insertAt) + insertion + block.slice(insertAt)
+			} else {
+				newBlock = `${block.replace(/\s*$/, '')}\n${insertion}`
+			}
+			if (decision.note) {
+				newBlock = `${newBlock.replace(/\s*$/, '')}\n\n### Validation Note\n${decision.note}\n`
+			}
+		}
+		out = out.slice(0, finding.start) + newBlock + out.slice(finding.end)
+	}
+	return out
+}
+
+/**
+ * Main command entry point.
+ */
+export function cmdValidateEvidence(findingsDir, extraArgs) {
+	if (!findingsDir)
+		return err('Usage: apt-tools validate-evidence <findings-dir> --worktree <path>')
+	const dir = resolve(findingsDir)
+	if (!existsSync(dir)) return err(`Findings directory not found: ${dir}`)
+
+	const flags = parseFlags(extraArgs)
+	const worktree = flags.get('worktree')
+	if (!worktree) return err('Required: --worktree <path>')
+	const worktreeRoot = resolve(worktree)
+	if (!existsSync(worktreeRoot)) return err(`Worktree not found: ${worktreeRoot}`)
+
+	const files = readdirSync(dir).filter((f) => f.endsWith('.md'))
+	const summary = {
+		status: 'ok',
+		command: 'validate-evidence',
+		findings_dir: dir,
+		worktree: worktreeRoot,
+		validated: 0,
+		rejected: 0,
+		rejections: [],
+		processed_files: [],
+	}
+
+	for (const f of files) {
+		const full = join(dir, f)
+		const md = readFileSync(full, 'utf-8')
+		const findings = parseFindingsMarkdown(md)
+		if (findings.length === 0) continue
+
+		const decisions = []
+		for (const finding of findings) {
+			const decision = checkFinding(finding, worktreeRoot)
+			decisions.push({ finding, decision })
+			if (decision.status === 'confirmed_valid') {
+				summary.validated++
+			} else {
+				summary.rejected++
+				summary.rejections.push({
+					id: finding.id,
+					file: f,
+					finding_file: finding.file,
+					status: decision.status,
+					reason: decision.note,
+				})
+			}
+		}
+
+		const rewritten = stampFindings(md, decisions)
+		if (rewritten !== md) {
+			writeFileSync(full, rewritten, 'utf-8')
+		}
+		summary.processed_files.push(f)
+	}
+
+	return ok(summary)
+}