docguard-cli 0.30.1 → 0.31.0

package/cli/shared-diff.mjs

@@ -0,0 +1,209 @@
+/**
+ * Shared Unified-Diff Parser + Tokenizer — zero-dependency foundation for the
+ * change-driven detectors added in v0.31.0.
+ *
+ * ONE parser, consumed by four features so they never re-implement diff
+ * scraping (each previously would have grepped `git diff` output ad hoc):
+ *   - diff-overlap suspicion (validators/diff-suspicion.mjs): does a doc's
+ *     wording overlap tokens that were DELETED/REPLACED-OLD in the code diff?
+ *   - reference-existence (validators/reference-existence.mjs): which symbols
+ *     left the tree between two revisions.
+ *   - impact blast-radius (commands/impact.mjs): which docs cite changed code.
+ *   - structured-diff staging (commands/verify.mjs): hand agents an ordered
+ *     replace/delete/add representation (the CARL-CCI "activity-labeled diff"
+ *     shown to beat raw-text diffs — arXiv 2512.19883) instead of a raw patch.
+ *
+ * Pure Node built-ins. No git here — this operates on diff TEXT a caller
+ * already produced (see shared-git.getDiffSpans). That keeps it unit-testable
+ * without a repo and reusable on any unified-diff string.
+ */
+
+// ── Tokenizer ────────────────────────────────────────────────────────────────
+// Identifier-aware: keeps `getUserById`, `user_id`, `UserService` whole, then
+// also emits their sub-words so a doc saying "user id" still overlaps code
+// token `user_id`. Deterministic, lowercase, stopword-filtered.
+
+const STOPWORDS = new Set([
+  'the', 'a', 'an', 'and', 'or', 'but', 'if', 'then', 'else', 'for', 'of', 'to',
+  'in', 'on', 'at', 'by', 'as', 'is', 'are', 'was', 'were', 'be', 'been', 'this',
+  'that', 'these', 'those', 'it', 'its', 'with', 'from', 'not', 'no', 'we', 'you',
+  'return', 'returns', 'const', 'let', 'var', 'function', 'class', 'import',
+  'export', 'default', 'new', 'true', 'false', 'null', 'void', 'public', 'private',
+]);
+
+/**
+ * Split identifiers into sub-words: getUserById → [get,user,by,id];
+ * user_id → [user,id]; HTTPServer → [http,server].
+ */
+export function splitIdentifier(id) {
+  return String(id)
+    // camelCase / PascalCase / ACRONYMBoundary
+    .replace(/([a-z0-9])([A-Z])/g, '$1 $2')
+    .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2')
+    // snake_case, kebab-case, dot.paths
+    .replace(/[_\-.]+/g, ' ')
+    .toLowerCase()
+    .split(/\s+/)
+    .filter(Boolean);
+}
+
+/**
+ * Tokenize free text OR code into a lowercase word set. Keeps whole identifiers
+ * AND their sub-words. `min` drops tokens shorter than it (default 3) to cut
+ * noise; identifiers below `min` after splitting are still dropped.
+ *
+ * Returns an array (call-site decides Set vs list). Deduped, order-preserving.
+ */
+export function tokenize(text, { min = 3, keepStopwords = false } = {}) {
+  const out = [];
+  const seen = new Set();
+  const raw = String(text).match(/[A-Za-z_][A-Za-z0-9_.-]*/g) || [];
+  for (const word of raw) {
+    // the whole identifier (lowercased) …
+    const whole = word.toLowerCase();
+    for (const t of [whole, ...splitIdentifier(word)]) {
+      if (t.length < min) continue;
+      if (!keepStopwords && STOPWORDS.has(t)) continue;
+      if (seen.has(t)) continue;
+      seen.add(t);
+      out.push(t);
+    }
+  }
+  return out;
+}
+
+// ── Unified-diff parser ──────────────────────────────────────────────────────
+
+/**
+ * Parse a unified diff (git diff / diff -u) into structured file entries.
+ * Handles multi-file diffs, adds/deletes (/dev/null), renames, and the
+ * "" marker.
+ *
+ * Returns: [{ oldPath, newPath, status, hunks: [{ oldStart, newStart,
+ *   lines: [{ op: ' '|'-'|'+', text }] }] }]
+ * status ∈ 'modified' | 'added' | 'deleted' | 'renamed'.
+ */
+export function parseUnifiedDiff(diffText) {
+  const files = [];
+  if (!diffText) return files;
+  const lines = String(diffText).split('\n');
+  let cur = null;
+  let hunk = null;
+
+  const pushHunkHeader = (m) => {
+    hunk = { oldStart: parseInt(m[1], 10) || 0, newStart: parseInt(m[2], 10) || 0, lines: [] };
+    cur.hunks.push(hunk);
+  };
+
+  for (const line of lines) {
+    if (line.startsWith('diff --git')) {
+      // a/<old> b/<new> — quotes possible but rare; keep it simple.
+      const m = line.match(/^diff --git a\/(.+?) b\/(.+)$/);
+      cur = { oldPath: m ? m[1] : null, newPath: m ? m[2] : null, status: 'modified', hunks: [] };
+      files.push(cur);
+      hunk = null;
+      continue;
+    }
+    if (!cur) continue; // ignore any preamble before the first file
+    if (line.startsWith('rename from ')) { cur.status = 'renamed'; cur.oldPath = line.slice(12); continue; }
+    if (line.startsWith('rename to ')) { cur.status = 'renamed'; cur.newPath = line.slice(10); continue; }
+    if (line.startsWith('--- ')) {
+      const p = line.slice(4);
+      if (p === '/dev/null') cur.status = 'added';
+      else cur.oldPath = p.replace(/^a\//, '');
+      continue;
+    }
+    if (line.startsWith('+++ ')) {
+      const p = line.slice(4);
+      if (p === '/dev/null') cur.status = 'deleted';
+      else cur.newPath = p.replace(/^b\//, '');
+      continue;
+    }
+    const hm = line.match(/^@@ -(\d+)(?:,\d+)? \+(\d+)(?:,\d+)? @@/);
+    if (hm) { pushHunkHeader(hm); continue; }
+    if (!hunk) continue;
+    if (line.startsWith('\\')) continue; // ""
+    const op = line[0];
+    if (op === ' ' || op === '+' || op === '-') {
+      hunk.lines.push({ op, text: line.slice(1) });
+    }
+  }
+  return files;
+}
+
+/**
+ * Decompose a file's hunks into an ORDERED activity list — the CARL-CCI
+ * "activity-labeled" representation. Consecutive removed/added runs are grouped:
+ *   run of '-' then '+'  → { type: 'replace', del: [...], add: [...] }
+ *   run of '-' only      → { type: 'delete', del: [...] }
+ *   run of '+' only      → { type: 'add', add: [...] }
+ * Context lines are the separators and are not emitted (they're not a change).
+ *
+ * This is what makes staged agent tasks legible: the agent sees "these lines
+ * were replaced by those", not a flat blob.
+ */
+export function activityLabeledDiff(fileDiff) {
+  const acts = [];
+  for (const h of fileDiff.hunks || []) {
+    let del = [];
+    let add = [];
+    const flush = () => {
+      if (del.length && add.length) acts.push({ type: 'replace', del, add });
+      else if (del.length) acts.push({ type: 'delete', del });
+      else if (add.length) acts.push({ type: 'add', add });
+      del = []; add = [];
+    };
+    for (const ln of h.lines) {
+      if (ln.op === '-') {
+        if (add.length) flush(); // an add run ended before this delete → boundary
+        del.push(ln.text);
+      } else if (ln.op === '+') {
+        add.push(ln.text);
+      } else {
+        flush(); // context terminates the current activity
+      }
+    }
+    flush();
+  }
+  return acts;
+}
+
+/**
+ * Tokens that LEFT the code in this file diff — union over every '-' line
+ * (both pure deletes and the "old" side of replaces). This is the
+ * `deleted ∪ replaceOld` span the outdated-comment research keys on: a doc
+ * that still talks about these tokens is a drift suspect.
+ */
+export function removedTokens(fileDiff, opts) {
+  const set = new Set();
+  for (const h of fileDiff.hunks || []) {
+    for (const ln of h.lines) {
+      if (ln.op === '-') for (const t of tokenize(ln.text, opts)) set.add(t);
+    }
+  }
+  return set;
+}
+
+/** Tokens that were ADDED ('+' lines) — the new-side span. */
+export function addedTokens(fileDiff, opts) {
+  const set = new Set();
+  for (const h of fileDiff.hunks || []) {
+    for (const ln of h.lines) {
+      if (ln.op === '+') for (const t of tokenize(ln.text, opts)) set.add(t);
+    }
+  }
+  return set;
+}
+
+/**
+ * Overlap score between a doc's tokens and a set of change tokens: the count
+ * and the shared tokens themselves (for explainable findings). Deterministic.
+ */
+export function tokenOverlap(docTokens, changeTokenSet) {
+  const shared = [];
+  const seen = new Set();
+  for (const t of docTokens) {
+    if (changeTokenSet.has(t) && !seen.has(t)) { seen.add(t); shared.push(t); }
+  }
+  return { count: shared.length, shared };
+}

package/cli/shared-git.mjs

@@ -148,6 +148,99 @@ export function changedFilesSince(dir, ref = 'HEAD~1') {
   }
 }
 
+/**
+ * Return the raw unified-diff TEXT between `ref` and HEAD, restricted to code
+ * files (docs are excluded — a doc changing is not a code change that could
+ * make OTHER docs stale). Consumed by shared-diff.parseUnifiedDiff.
+ *
+ * `-U0`? No — we want a few lines of context so the parser can group activities
+ * and callers can see surrounding tokens; default 3 is fine. Returns '' on
+ * error / no diff. Caps output at ~5MB so a giant refactor can't OOM the CLI.
+ */
+export function getDiffText(dir, ref = 'HEAD~1', pathspec = null) {
+  try {
+    const args = ['diff', '--no-color', '--no-ext-diff', ref, 'HEAD'];
+    if (pathspec && pathspec.length) args.push('--', ...pathspec);
+    const raw = execFileSync('git', args, {
+      cwd: dir, encoding: 'utf-8',
+      stdio: ['pipe', 'pipe', 'ignore'],
+      maxBuffer: 1024 * 1024 * 5,
+    });
+    return raw || '';
+  } catch {
+    return '';
+  }
+}
+
+/**
+ * Read a file's contents AS OF a given revision (e.g. the commit where a doc
+ * was last touched), following the `<rev>:<path>` git addressing. Returns null
+ * when the path didn't exist at that rev, the rev is unknown, or git is
+ * unavailable — callers treat null as "no prior snapshot to compare".
+ *
+ * This is the backbone of the two-revision reference-existence check: read the
+ * source at the doc's last-updated commit vs HEAD and diff symbol presence.
+ */
+export function fileContentAtRev(dir, rev, filePath) {
+  try {
+    const raw = execFileSync(
+      'git',
+      ['show', `${rev}:${filePath}`],
+      { cwd: dir, encoding: 'utf-8', stdio: ['pipe', 'pipe', 'ignore'], maxBuffer: 1024 * 1024 * 10 }
+    );
+    return raw;
+  } catch {
+    return null;
+  }
+}
+
+// Default source globs for symbol-existence grep (any-language).
+export const CODE_GLOBS = [
+  '*.ts', '*.tsx', '*.js', '*.jsx', '*.mjs', '*.cjs', '*.py', '*.go', '*.rs',
+  '*.java', '*.kt', '*.rb', '*.php', '*.cs', '*.swift', '*.scala', '*.dart', '*.c', '*.cpp', '*.h',
+];
+
+/**
+ * True if `symbol` appears as a whole word in the source tree AS OF `rev`.
+ * Uses `git grep -w -F` (fixed string, word boundary) at the given revision —
+ * exactly the "whole-word, case-sensitive, exact string match" the two-revision
+ * outdated-reference method specifies (arXiv 2212.01479). Restricted to code
+ * globs so a symbol still named in prose/docs doesn't count as "present".
+ *
+ * Returns false when absent, the rev is unknown, or git is unavailable — the
+ * caller pairs two calls (doc's last-update rev vs HEAD) to detect present→gone.
+ */
+export function symbolExistsAtRev(dir, symbol, rev, pathspecs = CODE_GLOBS) {
+  try {
+    execFileSync(
+      'git',
+      ['grep', '-q', '-w', '-F', '-e', symbol, rev, '--', ...pathspecs],
+      { cwd: dir, stdio: ['pipe', 'ignore', 'ignore'] }
+    );
+    return true; // exit 0 → at least one match
+  } catch {
+    return false; // exit 1 → no match (or bad rev / no git)
+  }
+}
+
+/**
+ * Resolve the commit hash that last touched `filePath` (following renames), or
+ * null. Used to anchor "the revision when this doc was last updated" for the
+ * two-revision check without re-parsing getFileHistory at every call site.
+ */
+export function lastCommitHash(dir, filePath) {
+  try {
+    const raw = execFileSync(
+      'git',
+      ['log', '--follow', '-1', '--format=%H', '--', filePath],
+      { cwd: dir, encoding: 'utf-8', stdio: ['pipe', 'pipe', 'ignore'] }
+    ).trim();
+    return raw || null;
+  } catch {
+    return null;
+  }
+}
+
 /**
  * Resolve the absolute path to this repo's git hooks directory.
  *

package/cli/shared-ir.mjs

@@ -0,0 +1,81 @@
+/**
+ * Shared IR primitives — zero-dependency TF-IDF + cosine similarity.
+ *
+ * Backs IR-based traceability link recovery (feat 5): when a requirement has no
+ * exact `@req <ID>` annotation, rank candidate test/code artifacts by textual
+ * similarity (Vector Space Model, the canonical IR traceability technique —
+ * "basic linear algebra, no external services"). A requirement with NO
+ * candidate above threshold is a strong "unimplemented / untested" signal.
+ *
+ * Pure functions over token arrays; the caller tokenizes (we reuse the
+ * identifier-aware tokenizer from shared-diff so `getUserById` in code matches
+ * "get user by id" in a requirement).
+ */
+
+/** term → count for one document's tokens. */
+export function termFreq(tokens) {
+  const tf = new Map();
+  for (const t of tokens) tf.set(t, (tf.get(t) || 0) + 1);
+  return tf;
+}
+
+/**
+ * Inverse document frequency across a corpus of token arrays.
+ * idf(t) = ln(N / (1 + df(t))) + 1  — smoothed so a term in every doc still
+ * carries a small positive weight (avoids all-zero vectors on tiny corpora).
+ */
+export function buildIdf(corpusTokenArrays) {
+  const N = corpusTokenArrays.length || 1;
+  const df = new Map();
+  for (const tokens of corpusTokenArrays) {
+    for (const t of new Set(tokens)) df.set(t, (df.get(t) || 0) + 1);
+  }
+  const idf = new Map();
+  for (const [t, d] of df) idf.set(t, Math.log(N / (1 + d)) + 1);
+  return idf;
+}
+
+/** TF-IDF vector (Map term→weight) for a document, given a prebuilt idf. */
+export function tfidfVector(tokens, idf) {
+  const tf = termFreq(tokens);
+  const vec = new Map();
+  const len = tokens.length || 1;
+  for (const [t, count] of tf) {
+    const w = idf.get(t);
+    if (w === undefined) continue; // term not in corpus idf → skip
+    vec.set(t, (count / len) * w); // normalized TF × IDF
+  }
+  return vec;
+}
+
+/** Cosine similarity of two sparse Map vectors. 0 when either is empty. */
+export function cosineSimilarity(a, b) {
+  if (a.size === 0 || b.size === 0) return 0;
+  // iterate the smaller for the dot product
+  const [small, large] = a.size <= b.size ? [a, b] : [b, a];
+  let dot = 0;
+  for (const [t, w] of small) {
+    const w2 = large.get(t);
+    if (w2 !== undefined) dot += w * w2;
+  }
+  if (dot === 0) return 0;
+  let na = 0; for (const w of a.values()) na += w * w;
+  let nb = 0; for (const w of b.values()) nb += w * w;
+  const denom = Math.sqrt(na) * Math.sqrt(nb);
+  return denom === 0 ? 0 : dot / denom;
+}
+
+/**
+ * Rank candidate documents against a query by cosine similarity.
+ * @param queryTokens tokens of the requirement text
+ * @param candidates  [{ id, tokens }]
+ * @returns [{ id, score }] sorted desc, scores in [0,1]
+ */
+export function rankBySimilarity(queryTokens, candidates) {
+  const corpus = [queryTokens, ...candidates.map(c => c.tokens)];
+  const idf = buildIdf(corpus);
+  const qv = tfidfVector(queryTokens, idf);
+  return candidates
+    .map(c => ({ id: c.id, score: cosineSimilarity(qv, tfidfVector(c.tokens, idf)) }))
+    .sort((x, y) => y.score - x.score);
+}

package/cli/validators/api-doc-smells.mjs

@@ -0,0 +1,143 @@
+/**
+ * API-Doc-Smells validator (APS001 Bloated, APS002 Lazy) — v0.31.0.
+ *
+ * Research: the API-documentation-smell taxonomy (Bloated, Excess Structural
+ * Info, Tangled, Fragmented, Lazy) with a 1,000-unit benchmark. The two smells
+ * with strong DETERMINISTIC detectors are Bloated (F1 0.90) and Lazy (F1 0.95),
+ * keyed on documentation length relative to the surface documented — no ML.
+ * The three semantic smells need BERT and are deliberately left to staged agent
+ * judgment (verify --semantic), matching DocGuard's split.
+ *
+ * We apply the length signals per "API documentation unit" = a markdown section
+ * whose HEADING is a code signature (an HTTP endpoint `GET /path`, a function
+ * `foo(...)`, or a backticked symbol). Prose-only sections are ignored — that's
+ * doc-quality.mjs's job (passive voice, readability); this is API-surface-specific.
+ *
+ *   Lazy   — a documented endpoint/method with (almost) no explanation: ≤ N
+ *            prose words of body. "Documented in name only."
+ *   Bloated— a single unit that is grossly over-documented: ≥ M words.
+ *
+ * All findings confidence:'low' / soft — a nudge to right-size the doc.
+ */
+
+import { existsSync, readFileSync, readdirSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { mkFinding, resultFromFindings } from '../findings.mjs';
+
+const HEADING = /^(#{1,6})\s+(.*)$/;
+// A heading that documents an API/code unit — NOT a prose section heading.
+// Precision (corpus-tuned): markdown headings routinely read "Some Words
+// (parenthetical note)", which naively looks like a call. A real signature has
+// NO space before `(` AND a code-shaped identifier (camelCase / snake_case /
+// dotted). This kills FPs like "Remediation Log (2026-03-17)", "Unit Tests
+// (vitest)", "4.1 Enrollment (Base Record)".
+const HTTP_SIG = /^`?\s*(GET|POST|PUT|DELETE|PATCH|HEAD|OPTIONS)\s+\/?`?\S/i;
+const FUNC_SIG = /(?:^|[\s`.])([A-Za-z_][A-Za-z0-9_]*)\((?:\)|[^)]*\))/; // identifier(...) no space
+const CODEY = /[a-z][A-Z]|_|\./;                     // camel/snake/dotted → code-shaped
+const BACKTICK_SIG = /^`[^`\s][^`]*`\s*$/;           // heading is exactly a `symbol`
+function isSignatureHeading(text) {
+  const t = text.trim();
+  if (HTTP_SIG.test(t)) return true;
+  if (BACKTICK_SIG.test(t)) return true;
+  const fm = t.match(FUNC_SIG);
+  if (fm && (CODEY.test(fm[1]) || /\(\s*\)/.test(t))) return true; // codey name OR empty-arg call foo()
+  return false;
+}
+
+// Prose words in a body, EXCLUDING fenced code blocks (a big code sample isn't
+// "explanation", so a unit with only code counts as Lazy).
+function proseWordCount(bodyLines) {
+  let inFence = false;
+  let words = 0;
+  for (const line of bodyLines) {
+    if (/^\s*```/.test(line)) { inFence = !inFence; continue; }
+    if (inFence) continue;
+    const m = line.trim().match(/[A-Za-z0-9][A-Za-z0-9'-]*/g);
+    if (m) words += m.length;
+  }
+  return words;
+}
+function totalWordCount(bodyLines) {
+  let words = 0;
+  for (const line of bodyLines) {
+    const m = line.match(/[A-Za-z0-9][A-Za-z0-9'-]*/g);
+    if (m) words += m.length;
+  }
+  return words;
+}
+
+// Split markdown into signature-headed units: { heading, level, line, body[] }.
+function extractUnits(content) {
+  const lines = content.split('\n');
+  const units = [];
+  let cur = null;
+  for (let i = 0; i < lines.length; i++) {
+    const hm = lines[i].match(HEADING);
+    if (hm) {
+      // close current unit at the next heading of same-or-higher level
+      if (cur && hm[1].length <= cur.level) { units.push(cur); cur = null; }
+      if (!cur && isSignatureHeading(hm[2])) {
+        cur = { heading: hm[2].trim(), level: hm[1].length, line: i + 1, body: [] };
+        continue;
+      }
+      if (cur) { cur.body.push(lines[i]); }
+      continue;
+    }
+    if (cur) cur.body.push(lines[i]);
+  }
+  if (cur) units.push(cur);
+  return units;
+}
+
+export function validateApiDocSmells(projectDir, config = {}) {
+  const cfg = config.apiDocSmells || {};
+  const lazyMax = Number.isInteger(cfg.lazyMaxWords) ? cfg.lazyMaxWords : 6;
+  const bloatedMin = Number.isInteger(cfg.bloatedMinWords) ? cfg.bloatedMinWords : 300;
+
+  const docsDir = resolve(projectDir, 'docs-canonical');
+  if (!existsSync(docsDir)) {
+    return resultFromFindings([], { passed: 0, total: 0, applicable: false });
+  }
+  let docFiles = [];
+  try { docFiles = readdirSync(docsDir).filter(f => f.endsWith('.md')); } catch { /* skip */ }
+
+  const findings = [];
+  let unitCount = 0;
+  for (const f of docFiles) {
+    let content;
+    try { content = readFileSync(resolve(docsDir, f), 'utf-8'); } catch { continue; }
+    const units = extractUnits(content);
+    for (const u of units) {
+      unitCount++;
+      const prose = proseWordCount(u.body);
+      const total = totalWordCount(u.body);
+      if (prose <= lazyMax) {
+        findings.push(mkFinding({
+          code: 'APS002',
+          validator: 'api-doc-smells',
+          severity: 'warn',
+          confidence: 'low',
+          message: `${f}: "${u.heading.slice(0, 60)}" is documented in name only (${prose} words of explanation) — Lazy API doc.`,
+          location: { file: f, line: u.line },
+          suggestion: { summary: `Describe what "${u.heading.slice(0, 40)}" does, its params, return, and errors — not just its signature.` },
+        }));
+      } else if (total >= bloatedMin) {
+        findings.push(mkFinding({
+          code: 'APS001',
+          validator: 'api-doc-smells',
+          severity: 'warn',
+          confidence: 'low',
+          message: `${f}: "${u.heading.slice(0, 60)}" is ${total} words for one unit — Bloated API doc; trim to the essential contract.`,
+          location: { file: f, line: u.line },
+          suggestion: { summary: `Split or trim "${u.heading.slice(0, 40)}" — move examples/edge-cases elsewhere and keep the core contract.` },
+        }));
+      }
+    }
+  }
+
+  return resultFromFindings(findings, {
+    passed: unitCount - findings.length,
+    total: unitCount,
+    applicable: unitCount > 0,
+  });
+}