docguard-cli 0.29.0 → 0.30.0

package/cli/scanners/instruction-audit.mjs

@@ -0,0 +1,320 @@
+/**
+ * Instruction Audit scanner — drift/conflict audit WITHIN agent instruction
+ * files (AGENTS.md, CLAUDE.md). Inspired by spec-kit's MemoryLint.
+ *
+ * Agent instruction files rot in a specific way: rules get duplicated across
+ * AGENTS.md and CLAUDE.md, then one copy is edited and the other isn't — an
+ * agent reading both now holds two contradictory orders and silently picks
+ * one. Rules also point at files that were renamed away, and at docguard
+ * subcommands that no longer exist. None of the doc↔code validators see this
+ * class: the drift is doc↔doc, inside the instruction layer itself.
+ *
+ * DocGuard's split applies (sibling of semantic-claims.mjs):
+ *   - DETERMINISTIC: extract the rules, then flag what string logic can prove
+ *     — exact-normalized duplicates, direct never/always negation pairs,
+ *     pointers to nonexistent files, references to unknown docguard commands.
+ *   - LLM JUDGMENT: rule pairs in the same topical cluster (≥2 shared
+ *     significant stems) become tasks for the agent running
+ *     `docguard verify --instructions` — "do these contradict in practice?".
+ *     Cross-file pairs are prioritized: AGENTS-vs-CLAUDE divergence is the
+ *     classic drift.
+ *
+ * Precision over recall: a line is only a rule when it carries an
+ * imperative/modal signal (must/never/always/…); a pointer is only checked
+ * when it lexes as a relative file path; a docguard command is only checked
+ * inside backticks (prose like "docguard should never…" is not an
+ * invocation). Files generated by `docguard agents --sync` (they carry the
+ * docguard:agents-sync marker) are skipped — auditing a generated mirror
+ * against its source would flag every rule as a duplicate.
+ *
+ * Zero npm dependencies — pure Node.js built-ins.
+ */
+
+import { existsSync, readFileSync, readdirSync } from 'node:fs';
+import { resolve, join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+const INSTRUCTION_FILES = ['AGENTS.md', 'CLAUDE.md'];
+const GENERATED_MARKER = 'docguard:agents-sync';
+
+/** A line/sentence is only a rule when it carries an imperative/modal signal. */
+const SIGNAL_RE = /\b(must|never|always|do not|don't|should|require[sd]?|forbid|only)\b/i;
+
+const MAX_RULES = 400;
+const MAX_TASKS = 40;
+
+// ── Normalization ───────────────────────────────────────────────────────────
+
+/** lowercase, drop apostrophes (don't → dont), all other punctuation → space. */
+function normalizeRule(text) {
+  return text
+    .toLowerCase()
+    .replace(/['’]/g, '')
+    .replace(/[^a-z0-9\s]/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+// Negation tokens (counted for parity) vs the full polarity set (stripped for
+// comparison). "do not" must precede "not"/"do" in the alternation so it
+// matches as one token.
+const NEG_RE = /\b(never|dont|do not|not)\b/g;
+const POLARITY_RE = /\b(never|dont|do not|not|always|must|do|should|shall)\b/g;
+
+const negationCount = (norm) => (norm.match(NEG_RE) || []).length;
+const stripPolarity = (norm) => norm.replace(POLARITY_RE, ' ').replace(/\s+/g, ' ').trim();
+
+// ── Rule extraction (deterministic) ─────────────────────────────────────────
+
+/**
+ * Parse AGENTS.md + CLAUDE.md into rules: list items or paragraph sentences
+ * carrying an imperative/modal signal. Skips fenced code, tables, HTML
+ * comments, and files generated by `docguard agents --sync`.
+ * @returns {Array<{ file, line, section, text }>}
+ */
+export function extractInstructionRules(projectDir) {
+  const rules = [];
+  for (const file of INSTRUCTION_FILES) {
+    let content;
+    try { content = readFileSync(resolve(projectDir, file), 'utf-8'); } catch { continue; }
+    if (content.includes(GENERATED_MARKER)) continue; // generated mirror of AGENTS.md — audit the source, not the copy
+
+    const lines = content.split('\n');
+    let section = '';
+    let inFence = false;
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i];
+      if (/^\s*```/.test(line)) { inFence = !inFence; continue; }
+      if (inFence) continue; // rules in code samples are examples, not orders
+      const h = line.match(/^(#{2,3})\s+(.+)$/);
+      if (h) { section = h[2].trim(); continue; }
+      const t = line.trim();
+      if (!t || t.startsWith('#') || t.startsWith('<!--') || t.startsWith('|')) continue;
+
+      // A list item is one candidate rule; a paragraph line splits into sentences.
+      const li = t.match(/^(?:[-*+]|\d+[.)])\s+(.*)$/);
+      const candidates = li ? [li[1]] : t.replace(/^>\s*/, '').split(/(?<=[.!?])\s+/);
+      for (const cand of candidates) {
+        const text = cand.trim();
+        if (text.length < 8 || !SIGNAL_RE.test(text)) continue;
+        rules.push({ file, line: i + 1, section, text: text.slice(0, 200) });
+        if (rules.length >= MAX_RULES) return rules;
+      }
+    }
+  }
+  return rules;
+}
+
+// ── Deterministic findings ──────────────────────────────────────────────────
+
+const PATH_EXTS = 'md|mjs|cjs|js|ts|tsx|jsx|json|ya?ml|py|sh|toml|txt|rs|go|css|html';
+// Backticked token: no spaces, lexes as a relative path with a known extension.
+const PATH_LIKE_RE = new RegExp(`^[\\w.-][\\w./-]*\\.(?:${PATH_EXTS})$`, 'i');
+// Bare (unbackticked) token: requires a directory separator for precision.
+const BARE_PATH_RE = new RegExp(`(?:^|[\\s("'])([\\w.-]+\\/[\\w./-]+\\.(?:${PATH_EXTS}))\\b`, 'gi');
+const BACKTICK_RE = /`([^`]+)`/g;
+const DOCGUARD_CMD_RE = /\bdocguard\s+([a-z][a-z0-9-]*)\b/g;
+
+/** File-path candidates referenced by a rule (anchors/line refs stripped). */
+function pathCandidates(text) {
+  const found = new Set();
+  BACKTICK_RE.lastIndex = 0;
+  let m;
+  while ((m = BACKTICK_RE.exec(text)) !== null) {
+    const tok = m[1].replace(/[#:].*$/, '').trim();
+    if (!tok.includes(' ') && PATH_LIKE_RE.test(tok)) found.add(tok);
+  }
+  BARE_PATH_RE.lastIndex = 0;
+  while ((m = BARE_PATH_RE.exec(text)) !== null) found.add(m[1].replace(/[#:].*$/, ''));
+  return [...found];
+}
+
+/**
+ * Known docguard subcommands: cli/commands/*.mjs basenames + permanent
+ * aliases. Read from disk at runtime so the list can't drift from the code.
+ * Returns null when unreadable — the caller then SKIPS the check (a wrong
+ * "unknown command" is worse than a missed one).
+ */
+function knownDocguardCommands() {
+  try {
+    const names = readdirSync(join(__dirname, '..', 'commands'))
+      .filter(f => f.endsWith('.mjs'))
+      .map(f => f.slice(0, -'.mjs'.length));
+    return new Set([...names, 'audit', 'dx']);
+  } catch { return null; }
+}
+
+/**
+ * The findings string logic can prove — no LLM involved.
+ * @returns {{ duplicates, negations, stalePointers, staleCommands }}
+ */
+export function findDeterministicFindings(rules, projectDir) {
+  // duplicates: exact-normalized matches, within or across files.
+  const byNorm = new Map();
+  for (const r of rules) {
+    const norm = normalizeRule(r.text);
+    if (!norm) continue;
+    if (!byNorm.has(norm)) byNorm.set(norm, []);
+    byNorm.get(norm).push(r);
+  }
+  const duplicates = [...byNorm.entries()]
+    .filter(([, rs]) => rs.length >= 2)
+    .map(([normalized, rs]) => ({ normalized, rules: rs }));
+
+  // direct-negation pairs: identical after stripping polarity tokens, with
+  // opposite negation-count parity ("never use tabs" vs "always use tabs").
+  const negations = [];
+  const byStripped = new Map();
+  for (const r of rules) {
+    const norm = normalizeRule(r.text);
+    const stripped = stripPolarity(norm);
+    if (stripped.split(' ').length < 2) continue; // "never" vs "always" alone proves nothing
+    if (!byStripped.has(stripped)) byStripped.set(stripped, []);
+    byStripped.get(stripped).push({ r, norm, parity: negationCount(norm) % 2 });
+  }
+  for (const [stripped, group] of byStripped) {
+    if (group.length < 2) continue;
+    for (let i = 0; i < group.length; i++) {
+      for (let j = i + 1; j < group.length; j++) {
+        const A = group[i], B = group[j];
+        if (A.parity === B.parity || A.norm === B.norm) continue; // same polarity → duplicate territory, not a conflict
+        negations.push({ a: A.r, b: B.r, common: stripped });
+      }
+    }
+  }
+
+  // stale pointers: referenced file paths that don't exist in the repo.
+  const stalePointers = [];
+  const seenPtr = new Set();
+  for (const r of rules) {
+    for (const p of pathCandidates(r.text)) {
+      const key = `${r.file}:${r.line}:${p}`;
+      if (seenPtr.has(key)) continue;
+      seenPtr.add(key);
+      if (!existsSync(resolve(projectDir, p))) {
+        stalePointers.push({ file: r.file, line: r.line, section: r.section, text: r.text, path: p });
+      }
+    }
+  }
+
+  // stale commands: `docguard <cmd>` (backticked — an invocation, not prose)
+  // where <cmd> is not a known command.
+  const staleCommands = [];
+  const known = knownDocguardCommands();
+  if (known) {
+    const seenCmd = new Set();
+    for (const r of rules) {
+      BACKTICK_RE.lastIndex = 0;
+      let span;
+      while ((span = BACKTICK_RE.exec(r.text)) !== null) {
+        DOCGUARD_CMD_RE.lastIndex = 0;
+        let m;
+        while ((m = DOCGUARD_CMD_RE.exec(span[1])) !== null) {
+          const command = m[1];
+          const key = `${r.file}:${r.line}:${command}`;
+          if (known.has(command) || seenCmd.has(key)) continue;
+          seenCmd.add(key);
+          staleCommands.push({ file: r.file, line: r.line, section: r.section, text: r.text, command });
+        }
+      }
+    }
+  }
+
+  return { duplicates, negations, stalePointers, staleCommands };
+}
+
+// ── LLM tasks (topical-cluster pairs) ───────────────────────────────────────
+
+// Function words + polarity/signal tokens. Content verbs (use, run, commit,
+// write, read…) stay significant — "never use tabs" / "always use spaces"
+// should cluster on use+indentation, not be filtered to nothing.
+const STOPWORDS = new Set([
+  'the', 'a', 'an', 'to', 'of', 'in', 'for', 'and', 'or', 'with', 'without',
+  'on', 'at', 'by', 'from', 'as', 'is', 'are', 'be', 'been', 'being', 'it',
+  'its', 'this', 'that', 'these', 'those', 'you', 'your', 'we', 'our', 'all',
+  'any', 'each', 'every', 'when', 'where', 'while', 'if', 'then', 'than',
+  'so', 'but', 'not', 'no', 'never', 'always', 'must', 'should', 'shall',
+  'do', 'dont', 'does', 'did', 'done', 'can', 'cannot', 'cant', 'may',
+  'might', 'will', 'would', 'could', 'only', 'before', 'after', 'into',
+  'over', 'under', 'via', 'per', 'also', 'ever', 'instead', 'rather',
+  'avoid', 'ensure', 'require', 'requires', 'required', 'forbid', 'forbidden',
+  'e', 'g', 'i', 'etc', 'please',
+]);
+
+/** Cheap suffix stemmer — consistency matters, not linguistics. */
+function stem(w) {
+  let s = w;
+  if (s.length > 5 && s.endsWith('ing')) s = s.slice(0, -3);
+  else if (s.length > 4 && (s.endsWith('ed') || s.endsWith('es'))) s = s.slice(0, -2);
+  else if (s.length > 3 && s.endsWith('s')) s = s.slice(0, -1);
+  if (s.length > 3 && s[s.length - 1] === s[s.length - 2]) s = s.slice(0, -1); // committ → commit
+  return s;
+}
+
+function significantStems(norm) {
+  const stems = new Set();
+  for (const w of norm.split(' ')) {
+    if (w.length < 3 || STOPWORDS.has(w) || /^\d+$/.test(w)) continue;
+    stems.add(stem(w));
+  }
+  return stems;
+}
+
+/**
+ * Rule pairs in the same topical cluster (≥2 shared significant stems) become
+ * agent judgment tasks. Pairs already proven by the deterministic pass (exact
+ * duplicates, direct negations) are excluded — no LLM needed there. Cross-file
+ * pairs first (AGENTS-vs-CLAUDE divergence is the classic drift), capped.
+ */
+export function buildInstructionAuditTasks(rules) {
+  const enriched = rules.map(r => {
+    const norm = normalizeRule(r.text);
+    return { r, norm, stripped: stripPolarity(norm), parity: negationCount(norm) % 2, stems: significantStems(norm) };
+  });
+
+  const pairs = [];
+  for (let i = 0; i < enriched.length; i++) {
+    for (let j = i + 1; j < enriched.length; j++) {
+      const A = enriched[i], B = enriched[j];
+      if (A.norm === B.norm) continue; // exact duplicate — deterministic finding
+      if (A.stripped === B.stripped && A.parity !== B.parity) continue; // direct negation — deterministic finding
+      const shared = [...A.stems].filter(s => B.stems.has(s));
+      if (shared.length < 2) continue;
+      pairs.push({ a: A.r, b: B.r, shared, crossFile: A.r.file !== B.r.file });
+    }
+  }
+
+  pairs.sort((p, q) =>
+    (q.crossFile - p.crossFile) ||
+    (q.shared.length - p.shared.length) ||
+    (p.a.line - q.a.line) || (p.b.line - q.b.line));
+
+  return pairs.slice(0, MAX_TASKS).map((p, i) => {
+    const at = (r) => `${r.file}:${r.line}${r.section ? ` (section "${r.section}")` : ''}`;
+    return {
+      id: `verify.instructions.${i + 1}`,
+      a: p.a,
+      b: p.b,
+      sharedTerms: p.shared,
+      crossFile: p.crossFile,
+      instruction: `Judge whether these two agent-instruction rules contradict in practice. Rule A — ${at(p.a)}: "${p.a.text}". Rule B — ${at(p.b)}: "${p.b.text}". They share the terms: ${p.shared.join(', ')}. If they conflict, report which rule should win, why, and which file to edit; if one merely duplicates the other, say which copy to delete; if they complement each other, say so.`,
+      confidence: 'requires-human',
+    };
+  });
+}
+
+// ── Entry point ─────────────────────────────────────────────────────────────
+
+/**
+ * Full instruction audit: extract rules, prove what string logic can prove,
+ * and stage the semantic-conflict judgments for the agent.
+ * @returns {{ rules, deterministic: {duplicates,negations,stalePointers,staleCommands}, tasks }}
+ */
+export function auditInstructions(projectDir, config = {}) {
+  const rules = extractInstructionRules(projectDir);
+  const deterministic = findDeterministicFindings(rules, projectDir);
+  const tasks = buildInstructionAuditTasks(rules);
+  return { rules, deterministic, tasks };
+}