docguard-cli 0.27.0 → 0.29.0

package/cli/shared-ignore.mjs

@@ -89,8 +89,8 @@ export function isNonProductPath(relPath, config = {}) {
  *
  * Returns [] if the file is missing or unreadable — never throws.
  */
-import { readFileSync, existsSync } from 'node:fs';
-import { resolve as resolvePath, relative as relativePath, sep } from 'node:path';
+import { readFileSync, existsSync, readdirSync, statSync } from 'node:fs';
+import { resolve as resolvePath, relative as relativePath, join as joinPath, sep } from 'node:path';
 
 /**
  * Project-relative path with POSIX (`/`) separators — the canonical form that
@@ -212,26 +212,62 @@ export function shouldIgnore(relPath, config, validatorKey) {
 }
 
 /**
- * Convert a glob pattern to a RegExp for POSITIVE matching.
- * Unlike globToRegex (used for ignore filtering), this anchors the match
- * to the full relative path from the project root.
+ * THE canonical anchored glob compiler (v0.29 consolidation).
  *
- * Supports: * (any chars except /), ** (any path segments), . (literal dot).
+ * The repo previously carried THREE glob→regex implementations (ignore-side
+ * `globToRegex` above, the old `globToMatchRegex` here, and a third private
+ * copy in metrics-consistency for collection counting) with subtly different
+ * feature sets — a maintenance hazard for exactly the drift class this tool
+ * detects in others. This is now the single anchored compiler; the ignore-side
+ * `globToRegex` deliberately stays separate because its UNanchored,
+ * boundary-substring semantics ("dir" matches at any depth) are a different
+ * contract, documented above, with its own bug history.
+ *
+ * Supports (superset of all prior anchored variants):
+ *   `**\/`  → zero or more path segments   → (?:.*\/)?
+ *   `**`    → any chars (incl. /)          → .*
+ *   `*`     → any chars except /           → [^/]*
+ *   `?`     → one char except /            → [^/]
+ *   `{a,b}` → alternation (non-nested)     → (?:a|b)
+ * Everything else is regex-escaped. Fully anchored: ^...$.
  *
- * @param {string} pattern - Glob pattern (e.g., "backend/**\/__tests__/**\/*.test.ts")
+ * @param {string} pattern - Glob pattern (e.g., "backend/**\/__tests__/**\/*.test.{ts,js}")
  * @returns {RegExp}
  */
-function globToMatchRegex(pattern) {
-  // Normalize: replace **/ with a placeholder that means "zero or more path segments"
-  let escaped = pattern
-    .replace(/\./g, '\\.')
-    .replace(/\*\*\//g, '§STARSTAR§')   // **/ → zero-or-more segments
-    .replace(/\*\*/g, '.*')             // standalone ** → any chars
-    .replace(/\*/g, '[^/]*')            // single * → any chars except /
-    .replace(/§STARSTAR§/g, '(.*/)?');  // **/ → optional path prefix
-  return new RegExp(`^${escaped}$`);
+export function compileGlob(pattern) {
+  const glob = String(pattern);
+  let re = '';
+  for (let i = 0; i < glob.length; i++) {
+    const ch = glob[i];
+    if (ch === '*') {
+      if (glob[i + 1] === '*') {
+        i++;
+        if (glob[i + 1] === '/') { re += '(?:.*/)?'; i++; }
+        else re += '.*';
+      } else {
+        re += '[^/]*';
+      }
+    } else if (ch === '?') {
+      re += '[^/]';
+    } else if (ch === '{') {
+      const end = glob.indexOf('}', i);
+      if (end > i) {
+        re += '(?:' + glob.slice(i + 1, end).split(',')
+          .map(s => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('|') + ')';
+        i = end;
+      } else {
+        re += '\\{';
+      }
+    } else {
+      re += ch.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    }
+  }
+  return new RegExp(`^${re}$`);
 }
 
+// Back-compat internal alias — globMatch below always used the anchored form.
+const globToMatchRegex = compileGlob;
+
 /**
  * Check if a relative path matches ANY of the given glob patterns.
  * Purpose-built for POSITIVE matching (e.g., "is this a test file?").
@@ -255,3 +291,99 @@ export function globMatch(relPath, patterns) {
   const regexes = patterns.map(p => globToMatchRegex(p));
   return regexes.some(r => r.test(relPath));
 }
+
+/**
+ * THE canonical recursive file walker (v0.29 consolidation).
+ *
+ * ~13 validators each carried a private recursive walker with its own copied
+ * IGNORE_DIRS set and its own error handling — thirteen chances for skip logic
+ * to disagree. This is the single shared implementation.
+ *
+ * Contract:
+ *   - Skips directory names in `ignoreDirs` (default: DEFAULT_IGNORE_DIRS) and,
+ *     by default, every dot-prefixed entry (files AND dirs — matches the
+ *     dominant prior behavior).
+ *   - Calls `callback(absPath)` for every regular file reached.
+ *   - NEVER throws. Unreadable entries invoke `onError(err, path)` if given.
+ *   - Returns `true` iff the walk was COMPLETE (no unreadable entries). Callers
+ *     computing counts MUST check this: a partial walk that silently under-
+ *     counts is how a "code has N" assertion becomes confidently wrong — the
+ *     tool's own worst failure mode.
+ *
+ * @param {string} dir - Absolute directory to walk
+ * @param {(absPath: string) => void} callback
+ * @param {{ignoreDirs?: Set<string>, skipDotEntries?: boolean, keepDot?: (entry: string) => boolean, onError?: (err: Error, path: string) => void}} [opts]
+ *   `keepDot` — exception predicate for dot entries that MUST be walked even
+ *   with skipDotEntries on. Load-bearing for e.g. the security validator
+ *   (must scan `.env`) and traceability (`.env*`, `.gitignore`, `.github/`).
+ * @returns {boolean} - true if every entry was readable
+ */
+export function walkFiles(dir, callback, opts = {}) {
+  const {
+    ignoreDirs = DEFAULT_IGNORE_DIRS,
+    skipDotEntries = true,
+    keepDot = null,
+    onError = null,
+  } = opts;
+  let entries;
+  try { entries = readdirSync(dir); } catch (err) {
+    if (onError) onError(err, dir);
+    return false;
+  }
+  let complete = true;
+  for (const entry of entries) {
+    if (ignoreDirs.has(entry)) continue;
+    if (skipDotEntries && entry.startsWith('.') && !(keepDot && keepDot(entry))) continue;
+    const full = joinPath(dir, entry);
+    let stat;
+    try { stat = statSync(full); } catch (err) {
+      if (onError) onError(err, full);
+      complete = false;
+      continue;
+    }
+    if (stat.isDirectory()) {
+      if (!walkFiles(full, callback, opts)) complete = false;
+    } else if (stat.isFile()) {
+      callback(full);
+    }
+  }
+  return complete;
+}
+
+/**
+ * Count files under `projectDir` matching an anchored glob (project-relative).
+ * The code-truth side of `config.collections` (metrics-consistency).
+ *
+ * Walks only from the glob's literal prefix — never the whole repo for a deep
+ * pattern. FAIL-SAFE BY CONTRACT:
+ *   - returns 0 when the base path doesn't exist (unresolved glob — caller skips);
+ *   - returns -1 when the walk was INCOMPLETE (permission-denied subtree, bad
+ *     pattern). Previously a partial walk silently under-counted, so a doc
+ *     saying "19 extractors" could be "corrected" to a wrong lower number.
+ * Callers must treat any value <= 0 as "don't assert".
+ *
+ * @param {string} projectDir
+ * @param {string} pattern - e.g. "src/extractors/*.py"
+ * @returns {number} match count, 0 = unresolved, -1 = unreliable
+ */
+export function countGlobFiles(projectDir, pattern) {
+  const norm = String(pattern).replace(/\\/g, '/').replace(/^\.\//, '');
+  if (!norm) return -1;
+  const baseSegs = [];
+  for (const seg of norm.split('/')) {
+    if (/[*?{]/.test(seg)) break;
+    baseSegs.push(seg);
+  }
+  const baseDir = resolvePath(projectDir, baseSegs.join('/') || '.');
+  if (!existsSync(baseDir)) return 0;
+  let re;
+  try { re = compileGlob(norm); } catch { return -1; }
+  try {
+    if (statSync(baseDir).isFile()) return re.test(norm) ? 1 : 0; // literal file pattern
+  } catch { return -1; }
+  let n = 0;
+  const complete = walkFiles(baseDir, (full) => {
+    if (re.test(relPosix(projectDir, full))) n++;
+  });
+  return complete ? n : -1;
+}

package/cli/shared.mjs

@@ -253,9 +253,53 @@ export const PROFILES = {
 };
 
 // ── .docguardignore Support ───────────────────────────────────────────────
-import { existsSync, readFileSync } from 'node:fs';
+import { existsSync, readFileSync, statSync } from 'node:fs';
 import { resolve, relative } from 'node:path';
 
+/**
+ * Conventional documentation-home directory names. A folder named one of these
+ * is unambiguously "docs DocGuard governs" — distinct from arbitrary markdown
+ * buried in a non-doc subdir (security/wolf-archive/, vendored toolkits), which
+ * the wu-whatsappinbox scoping fix deliberately excludes. We auto-track the
+ * former and never blanket-walk the latter.
+ */
+export const DEFAULT_DOC_DIRS = [
+  'docs', 'doc', 'documentation', 'docs-canonical', 'docs-implementation',
+  'guides', 'guide', 'handbook', 'manual', 'wiki', 'extensions',
+];
+
+/**
+ * Resolve the documentation-home directories for a project (relative dir paths,
+ * no trailing slash). Single source of truth so the claim scanner and the
+ * coverage map agree — "tracked" must mean "actually scanned," never a label
+ * the scanner ignores.
+ *
+ * Auto-detects the conventional doc-home names that actually exist at the root,
+ * plus the Docusaurus-style `website/docs`. `config.docs.dirs` EXTENDS that set
+ * (adds non-standard homes like a project's `wiki/`) rather than replacing it —
+ * the least-surprising model, since the whole point is to track MORE clearly-doc
+ * folders automatically. To EXCLUDE a conventional dir, use `.docguardignore`.
+ * NAMED dirs only — this never walks arbitrary subdirectories (that was the
+ * false-positive flood the scoping fix removed).
+ *
+ * @param {string} projectDir
+ * @param {object} [config]
+ * @returns {string[]} relative directory paths (e.g. ['docs', 'documentation'])
+ */
+export function resolveDocDirs(projectDir, config = {}) {
+  const isDir = (rel) => {
+    try { return statSync(resolve(projectDir, rel)).isDirectory(); } catch { return false; }
+  };
+  const out = new Set(DEFAULT_DOC_DIRS.filter(isDir));
+  if (isDir('website/docs')) out.add('website/docs');
+  const declared = config && config.docs && Array.isArray(config.docs.dirs) ? config.docs.dirs : [];
+  for (const d of declared) {
+    const norm = String(d).replace(/\\/g, '/').replace(/\/+$/, '');
+    if (norm) out.add(norm);
+  }
+  return [...out];
+}
+
 /**
  * Load ignore patterns from .docguardignore (like .gitignore).
  * Returns a function that checks if a relative path should be ignored.

package/cli/validators/api-surface.mjs

@@ -30,6 +30,7 @@ import { scanRoutesDeep } from '../scanners/routes.mjs';
 import { parseApiReferenceDoc, compareEndpoints, endpointKey } from '../scanners/api-doc.mjs';
 import { collectPackageJsons, getWorkspaceDirs } from '../shared-source.mjs';
 import { relPosix } from '../shared-ignore.mjs';
+import { mkFinding, resultFromFindings } from '../findings.mjs';
 
 const MAX_REPORTED = 15;
 const API_DOC = 'docs-canonical/API-REFERENCE.md';
@@ -208,10 +209,59 @@ export function computeApiSurfaceDrift(projectDir, config) {
   };
 }
 
+/**
+ * v0.28 (field report #4): diff the OpenAPI spec against the routes actually
+ * REGISTERED in code. When a spec exists, resolveApiSurface treats it as ground
+ * truth and the API-REFERENCE doc reconciles against it — so a spec that declares
+ * a phantom endpoint (no Express/Fastify route registers it) passes doc-vs-spec
+ * clean while the spec itself is wrong. This catches that.
+ *
+ * Conservative on purpose: only runs when code routes are actually scannable.
+ * If the scanner finds zero routes (unsupported framework, dynamically-registered
+ * routes), we can't tell "no route" from "scanner blind", so we skip rather than
+ * flag every spec endpoint as phantom. Reuses compareEndpoints so path-param /
+ * mount-prefix normalization matches the rest of the validator.
+ *
+ * @returns {{ applicable:boolean, specPath:string|null, routeCount:number,
+ *             matched:object[], specDeclaredNoRoute:object[], reason?:string }}
+ */
+export function computeSpecVsRouteDrift(projectDir, config) {
+  const specs = findAllOpenApiSpecs(projectDir, config);
+  if (specs.length === 0) {
+    return { applicable: false, specPath: null, routeCount: 0, matched: [], specDeclaredNoRoute: [], reason: 'no openapi spec' };
+  }
+  const spec = specs[0]; // authoritative (sourceRoot first, root last)
+  const framework = detectFramework(projectDir, config);
+  const routes = scanRoutesDeep(projectDir, { framework }, { openapi: { found: false } }, { config });
+  if (routes.length === 0) {
+    return { applicable: false, specPath: spec.relPath, routeCount: 0, matched: [], specDeclaredNoRoute: [], reason: 'no routes scannable' };
+  }
+  // documentedButAbsent = in the SPEC (first arg) but absent from the ROUTES
+  // (second arg) = spec-declares-but-no-route.
+  const cmp = compareEndpoints(
+    spec.endpoints.map(e => ({ method: e.method, path: e.path })),
+    routes.map(r => ({ method: r.method, path: r.path }))
+  );
+  return {
+    applicable: true,
+    specPath: spec.relPath,
+    routeCount: routes.length,
+    matched: cmp.matched,
+    specDeclaredNoRoute: cmp.documentedButAbsent,
+  };
+}
+
+// v0.29: migrated to structured findings (API001–API005). Messages are
+// byte-identical to the legacy strings — resultFromFindings derives the
+// errors/warnings arrays from the same findings; `fixes` and
+// `authoritativeSpec` are preserved.
 export function validateApiSurface(projectDir, config) {
-  const errors = [];
-  const warnings = [];
+  const findings = [];
   const fixes = [];
+  const trim = (arr) => {
+    const shown = arr.slice(0, MAX_REPORTED);
+    return { shown, extra: arr.length - shown.length };
+  };
 
   // v0.14-P2: when --changed-only scoping is active and NONE of the changed
   // files look like route/spec/controller files, this validator has nothing
@@ -222,7 +272,7 @@ export function validateApiSurface(projectDir, config) {
     const anyRouteFile = config.changedFiles.some(f => ROUTE_RE.test(f));
     if (!anyRouteFile) {
       return {
-        errors, warnings, passed: 0, total: 0, fixes,
+        errors: [], warnings: [], passed: 0, total: 0, fixes,
         applicable: false,
         note: 'no route/spec files in changed set',
       };
@@ -234,12 +284,17 @@ export function validateApiSurface(projectDir, config) {
   // choked on it. We fall back to code scanning (below), but the parse failure
   // is surfaced here rather than silently producing a clean "no surface" pass.
   for (const specPath of findUnparseableSpecs(projectDir, config)) {
-    warnings.push(
-      `OpenAPI spec ${specPath} declares paths but DocGuard parsed 0 endpoints from it ` +
-      `(likely an unsupported YAML feature — $ref, anchors, or folded scalars). ` +
-      `Falling back to code scanning; the spec's own endpoint list is unavailable. ` +
-      `Validate it with a full OpenAPI linter.`
-    );
+    findings.push(mkFinding({
+      code: 'API001',
+      validator: 'apiSurface',
+      severity: 'warn',
+      message: `OpenAPI spec ${specPath} declares paths but DocGuard parsed 0 endpoints from it ` +
+        `(likely an unsupported YAML feature — $ref, anchors, or folded scalars). ` +
+        `Falling back to code scanning; the spec's own endpoint list is unavailable. ` +
+        `Validate it with a full OpenAPI linter.`,
+      location: specPath,
+      suggestion: { kind: 'review', text: 'Validate the spec with a full OpenAPI linter (e.g. spectral) and simplify unsupported YAML features' },
+    }));
   }
 
   const drift = computeApiSurfaceDrift(projectDir, config);
@@ -250,38 +305,120 @@ export function validateApiSurface(projectDir, config) {
     const others = divergence.specs.slice(1).map(s => s.relPath).join(', ');
     const sample = divergence.divergent.slice(0, 8).join(', ');
     const more = divergence.divergent.length > 8 ? ` (+${divergence.divergent.length - 8} more)` : '';
-    warnings.push(
-      `Multiple OpenAPI specs disagree on ${divergence.divergent.length} endpoint(s): ` +
-      `${divergence.authoritative} (treated as authoritative) vs ${others}. Divergent: ${sample}${more}`
-    );
+    findings.push(mkFinding({
+      code: 'API002',
+      validator: 'apiSurface',
+      severity: 'warn',
+      message: `Multiple OpenAPI specs disagree on ${divergence.divergent.length} endpoint(s): ` +
+        `${divergence.authoritative} (treated as authoritative) vs ${others}. Divergent: ${sample}${more}`,
+      location: divergence.authoritative,
+      suggestion: { kind: 'review', text: 'Regenerate or delete the stale spec copy so every spec agrees on the endpoint set' },
+    }));
+  }
+
+  // ── #4: spec declares an endpoint with no registered route ──
+  // Independent of the API-REFERENCE doc — it checks the spec against code, so it
+  // runs even when no doc exists. Conservative (only when routes are scannable).
+  const specRoute = computeSpecVsRouteDrift(projectDir, config);
+  let specRouteTotal = 0;
+  let specRoutePassed = 0;
+  if (specRoute.applicable) {
+    specRouteTotal = specRoute.matched.length + specRoute.specDeclaredNoRoute.length;
+    specRoutePassed = specRoute.matched.length;
+    if (specRoute.specDeclaredNoRoute.length) {
+      const { shown, extra } = trim(specRoute.specDeclaredNoRoute);
+      for (const e of shown) {
+        findings.push(mkFinding({
+          code: 'API003',
+          validator: 'apiSurface',
+          severity: 'warn',
+          // "may be wrong" — the route scanner can be blind to dynamic
+          // registration, so this is a candidate false positive by design.
+          confidence: 'low',
+          message: `OpenAPI spec (${specRoute.specPath}) declares ${e.method} ${e.path} but no route registers it in code — ` +
+            `the spec may be wrong, and the API-REFERENCE doc reconciles clean against it, hiding the gap.`,
+          location: specRoute.specPath,
+          suggestion: { kind: 'review', text: 'Verify the endpoint: remove it from the spec if it no longer exists, or check whether the route is registered dynamically' },
+        }));
+      }
+      if (extra > 0) {
+        findings.push(mkFinding({
+          code: 'API003',
+          validator: 'apiSurface',
+          severity: 'warn',
+          confidence: 'low',
+          message: `…and ${extra} more spec-declared endpoint(s) with no registered route`,
+          location: specRoute.specPath,
+          suggestion: { kind: 'review', text: 'Verify each spec-declared endpoint against the registered routes' },
+        }));
+      }
+    }
   }
 
   if (!drift.applicable) {
-    // Nothing to validate against the API-REFERENCE doc.
-    return { errors, warnings, passed: 0, total: 0, fixes, authoritativeSpec: drift.source };
+    // Nothing to validate against the API-REFERENCE doc — but the spec-vs-route
+    // check above may still have produced findings.
+    return {
+      ...resultFromFindings(findings, { passed: specRoutePassed, total: specRouteTotal }),
+      fixes,
+      authoritativeSpec: drift.source || specRoute.specPath,
+    };
   }
 
   const { documentedButAbsent, presentButUndocumented, matched, confidence, source } = drift;
-  const total = matched.length + documentedButAbsent.length + presentButUndocumented.length;
-  const passed = matched.length;
-
-  const trim = (arr) => {
-    const shown = arr.slice(0, MAX_REPORTED);
-    return { shown, extra: arr.length - shown.length };
-  };
+  const total = matched.length + documentedButAbsent.length + presentButUndocumented.length + specRouteTotal;
+  const passed = matched.length + specRoutePassed;
 
   // documented-but-absent → deterministic remove-endpoint fixes
   if (documentedButAbsent.length) {
     const { shown, extra } = trim(documentedButAbsent);
     for (const e of shown) {
       const msg = `Documented endpoint not found in code: ${e.method} ${e.path} (${API_DOC})`;
-      if (confidence === 'spec') errors.push(msg);
-      else warnings.push(`${msg} [code-scan — verify]`);
+      if (confidence === 'spec') {
+        findings.push(mkFinding({
+          code: 'API004',
+          validator: 'apiSurface',
+          severity: 'error',
+          message: msg,
+          location: API_DOC,
+          suggestion: { kind: 'fix', text: 'Remove the dead endpoint from the doc', command: 'docguard fix --write' },
+        }));
+      } else {
+        findings.push(mkFinding({
+          code: 'API004',
+          validator: 'apiSurface',
+          severity: 'warn',
+          // The "[code-scan — verify]" suffix marks this as heuristic-only:
+          // the route scanner may simply not see the endpoint's registration.
+          confidence: 'low',
+          message: `${msg} [code-scan — verify]`,
+          location: API_DOC,
+          suggestion: { kind: 'review', text: 'Verify the endpoint really is gone from the code, then remove it from the doc' },
+        }));
+      }
     }
     if (extra > 0) {
       const tail = `…and ${extra} more documented endpoint(s) not found in code`;
-      if (confidence === 'spec') errors.push(tail);
-      else warnings.push(tail);
+      if (confidence === 'spec') {
+        findings.push(mkFinding({
+          code: 'API004',
+          validator: 'apiSurface',
+          severity: 'error',
+          message: tail,
+          location: API_DOC,
+          suggestion: { kind: 'fix', text: 'Remove the dead endpoints from the doc', command: 'docguard fix --write' },
+        }));
+      } else {
+        findings.push(mkFinding({
+          code: 'API004',
+          validator: 'apiSurface',
+          severity: 'warn',
+          confidence: 'low',
+          message: tail,
+          location: API_DOC,
+          suggestion: { kind: 'review', text: 'Verify each documented endpoint against the code, then prune the doc' },
+        }));
+      }
     }
     // Only spec-confirmed absences are safe to auto-remove.
     if (confidence === 'spec') {
@@ -295,10 +432,26 @@ export function validateApiSurface(projectDir, config) {
   if (presentButUndocumented.length) {
     const { shown, extra } = trim(presentButUndocumented);
     for (const e of shown) {
-      warnings.push(`Undocumented endpoint in code: ${e.method} ${e.path} — add it to ${API_DOC}`);
+      findings.push(mkFinding({
+        code: 'API005',
+        validator: 'apiSurface',
+        severity: 'warn',
+        message: `Undocumented endpoint in code: ${e.method} ${e.path} — add it to ${API_DOC}`,
+        location: API_DOC,
+        suggestion: { kind: 'fix', text: `Document the endpoint in ${API_DOC}` },
+      }));
+    }
+    if (extra > 0) {
+      findings.push(mkFinding({
+        code: 'API005',
+        validator: 'apiSurface',
+        severity: 'warn',
+        message: `…and ${extra} more undocumented endpoint(s) in code`,
+        location: API_DOC,
+        suggestion: { kind: 'fix', text: `Document the remaining endpoints in ${API_DOC}` },
+      }));
     }
-    if (extra > 0) warnings.push(`…and ${extra} more undocumented endpoint(s) in code`);
   }
 
-  return { errors, warnings, passed, total, fixes, authoritativeSpec: source };
+  return { ...resultFromFindings(findings, { passed, total }), fixes, authoritativeSpec: source };
 }