docguard-cli 0.23.0 → 0.25.0

package/cli/scanners/py-ast.mjs

@@ -0,0 +1,213 @@
+/**
+ * Python AST helpers — the "full support" parsing tier for Python, backed by
+ * the interpreter's OWN `ast` module (no npm/pip dependency: we shell out to
+ * the `python3` already on the developer's machine).
+ *
+ * Why a real parser here: the regex Python scanners match `@app.get("…")`
+ * decorators and `class X(BaseModel):` blocks line-by-line. That misses
+ * multi-line decorators, method-array Flask routes, and — most dangerously —
+ * undercounts a model's fields, which makes the data-model validators falsely
+ * PASS on stale docs. Python's `ast` gets every decorator and field exactly.
+ *
+ * Load model: OPTIONAL, exactly like the JS @babel/parser tier. If `python3`
+ * (or `python`) isn't on PATH, or the subprocess errors, every entry point here
+ * returns `null` and the callers transparently fall back to their regex (beta)
+ * tier. Python parsing never becomes load-bearing for the CLI to run.
+ */
+import { spawnSync } from 'node:child_process';
+
+// Cached interpreter probe: undefined = unchecked, null = unavailable,
+// string = the working command ('python3' or 'python').
+let _pyCmd;
+
+function pyCmd() {
+  if (_pyCmd !== undefined) return _pyCmd;
+  for (const cmd of ['python3', 'python']) {
+    try {
+      const r = spawnSync(cmd, ['-c', 'import ast,sys,json'], { encoding: 'utf-8', timeout: 4000 });
+      if (r.status === 0) { _pyCmd = cmd; return _pyCmd; }
+    } catch { /* try the next candidate */ }
+  }
+  _pyCmd = null;
+  return _pyCmd;
+}
+
+/** True when a usable Python 3 interpreter (with ast/json) is on PATH. */
+export function pyAstAvailable() {
+  return pyCmd() !== null;
+}
+
+// The extractor runs INSIDE python3. It reads newline-separated file paths on
+// stdin and writes a JSON array — one entry per file — to stdout. A file that
+// can't be parsed yields { ok: false } so the caller can fall back for THAT
+// file instead of silently treating it as "scanned, found nothing".
+//
+// Contains no backticks and no ${...}, so it embeds safely in a JS template.
+const PY_EXTRACTOR = `
+import ast, sys, json
+
+HTTP = {"get", "post", "put", "delete", "patch", "head", "options"}
+PYD_BASES = {"BaseModel", "SQLModel"}
+ORM_BASES = {"Base", "Model", "DeclarativeBase"}
+ORM_COLS = {"Column", "mapped_column", "relationship"}
+
+def str_of(node):
+    if isinstance(node, ast.Constant) and isinstance(node.value, str):
+        return node.value
+    return None
+
+def routes_from_func(fn):
+    out = []
+    doc = ast.get_docstring(fn) or ""
+    desc = doc.strip().split("\\n")[0] if doc else ""
+    for dec in fn.decorator_list:
+        if not isinstance(dec, ast.Call) or not isinstance(dec.func, ast.Attribute):
+            continue
+        method = dec.func.attr.lower()
+        if method in HTTP:
+            path = str_of(dec.args[0]) if dec.args else None
+            if path and path.startswith("/"):
+                out.append({"method": method.upper(), "path": path, "func": fn.name, "desc": desc})
+        elif method == "route":  # Flask: @app.route("/x", methods=["GET","POST"])
+            path = str_of(dec.args[0]) if dec.args else None
+            methods = ["GET"]
+            for kw in dec.keywords:
+                if kw.arg == "methods" and isinstance(kw.value, (ast.List, ast.Tuple)):
+                    ms = [str_of(e) for e in kw.value.elts]
+                    ms = [m.upper() for m in ms if m]
+                    if ms:
+                        methods = ms
+            if path and path.startswith("/"):
+                for m in methods:
+                    out.append({"method": m, "path": path, "func": fn.name, "desc": desc})
+    return out
+
+def base_names(cls):
+    names = []
+    for b in cls.bases:
+        if isinstance(b, ast.Name):
+            names.append(b.id)
+        elif isinstance(b, ast.Attribute):
+            names.append(b.attr)
+    return names
+
+def type_str(node):
+    f = getattr(ast, "unparse", None)  # ast.unparse is 3.9+; degrade to "" otherwise
+    if f is None or node is None:
+        return ""
+    try:
+        return f(node)
+    except Exception:
+        return ""
+
+def call_name(call):
+    fn = call.func
+    if isinstance(fn, ast.Attribute):
+        return fn.attr
+    if isinstance(fn, ast.Name):
+        return fn.id
+    return ""
+
+def fields_from_class(cls):
+    pyd, orm, rels = [], [], []
+    for stmt in cls.body:
+        # Pydantic: name: type [= default]
+        if isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name):
+            t = type_str(stmt.annotation)
+            has_none_default = isinstance(stmt.value, ast.Constant) and stmt.value.value is None
+            required = not ("Optional" in t or "None" in t or has_none_default)
+            pyd.append({"name": stmt.target.id, "type": t, "required": required})
+            if isinstance(stmt.value, ast.Call) and call_name(stmt.value) == "relationship" and stmt.value.args:
+                tgt = str_of(stmt.value.args[0])
+                if tgt:
+                    rels.append(tgt)
+        # SQLAlchemy: name = Column(Type, nullable=...) / mapped_column(...) / relationship("X")
+        elif isinstance(stmt, ast.Assign) and isinstance(stmt.value, ast.Call):
+            cname = call_name(stmt.value)
+            if cname in ORM_COLS:
+                t = ""
+                if stmt.value.args:
+                    a0 = stmt.value.args[0]
+                    if isinstance(a0, ast.Name):
+                        t = a0.id
+                    elif isinstance(a0, ast.Attribute):
+                        t = a0.attr
+                    elif isinstance(a0, ast.Call):
+                        t = call_name(a0)
+                required = True
+                for kw in stmt.value.keywords:
+                    if kw.arg == "nullable" and isinstance(kw.value, ast.Constant) and kw.value.value is True:
+                        required = False
+                for tgt in stmt.targets:
+                    if isinstance(tgt, ast.Name):
+                        orm.append({"name": tgt.id, "type": t, "required": required})
+                if cname == "relationship" and stmt.value.args:
+                    rel = str_of(stmt.value.args[0])
+                    if rel:
+                        rels.append(rel)
+    return pyd, orm, rels
+
+results = []
+for path in sys.stdin.read().splitlines():
+    path = path.strip()
+    if not path:
+        continue
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            tree = ast.parse(f.read(), filename=path)
+    except Exception:
+        results.append({"file": path, "ok": False})
+        continue
+    routes, schemas = [], []
+    for node in ast.walk(tree):
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            routes.extend(routes_from_func(node))
+        elif isinstance(node, ast.ClassDef):
+            bn = base_names(node)
+            pyd, orm, rels = fields_from_class(node)
+            if any(b in PYD_BASES for b in bn) and pyd:
+                schemas.append({"name": node.name, "fields": pyd, "kind": "pydantic", "rels": rels})
+            elif any(b in ORM_BASES for b in bn) and orm:
+                schemas.append({"name": node.name, "fields": orm, "kind": "sqlalchemy", "rels": rels})
+    results.append({"file": path, "ok": True, "routes": routes, "schemas": schemas})
+
+sys.stdout.write(json.dumps(results))
+`;
+
+/**
+ * Parse a batch of Python files in ONE python3 subprocess.
+ *
+ * @param {string[]} filePaths - absolute paths to .py files
+ * @returns {Object<string, {ok:boolean, routes?, schemas?}>|null}
+ *   A map keyed by the input path, or `null` when Python is unavailable / the
+ *   subprocess failed / output was unparseable (caller falls back to regex).
+ *   An empty input returns `{}` (nothing to do, but Python IS available).
+ */
+export function extractPythonFiles(filePaths) {
+  const cmd = pyCmd();
+  if (!cmd) return null;
+  if (!filePaths || filePaths.length === 0) return {};
+
+  let r;
+  try {
+    r = spawnSync(cmd, ['-c', PY_EXTRACTOR], {
+      input: filePaths.join('\n'),
+      encoding: 'utf-8',
+      maxBuffer: 64 * 1024 * 1024,
+      timeout: 30000,
+    });
+  } catch {
+    return null;
+  }
+  if (!r || r.status !== 0 || !r.stdout) return null;
+
+  let parsed;
+  try { parsed = JSON.parse(r.stdout); } catch { return null; }
+  if (!Array.isArray(parsed)) return null;
+
+  const byFile = {};
+  for (const entry of parsed) {
+    if (entry && entry.file) byFile[entry.file] = entry;
+  }
+  return byFile;
+}

package/cli/scanners/routes.mjs

@@ -8,12 +8,10 @@
 
 import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
 import { resolve, join, relative, basename, extname, dirname } from 'node:path';
-import { resolveSourceRoots } from '../shared-source.mjs';
-
-const IGNORE_DIRS = new Set([
-  'node_modules', '.git', '.next', 'dist', 'build', 'coverage',
-  '.cache', '__pycache__', '.venv', 'vendor', '.turbo',
-]);
+import { resolveSourceRoots, readScannable } from '../shared-source.mjs';
+import { DEFAULT_IGNORE_DIRS as IGNORE_DIRS, shouldIgnore, relPosix } from '../shared-ignore.mjs';
+import { extractJsRouteCalls, extractJsRouteObjects, extractJsMountsAndImports } from './js-ast.mjs';
+import { extractPythonFiles } from './py-ast.mjs';
 
 /**
  * Scan routes from source code with framework-aware parsing.
@@ -80,12 +78,16 @@ export function scanRoutesDeep(dir, stack, docTools, opts = {}) {
     routes.push(...scanFastAPIRoutes(dir));
   }
 
-  // Deduplicate by method+path
+  // Deduplicate by method+path, and honor .docguardignore / config.ignore so a
+  // fixtures dir with fake routes doesn't pollute the API surface. Filtering the
+  // RESULTS (route.file → project-relative) keeps the per-framework walkers as-is.
+  const cfg = opts.config || {};
   const seen = new Set();
   return routes.filter(r => {
     const key = `${r.method}:${r.path}`;
     if (seen.has(key)) return false;
     seen.add(key);
+    if (r.file && shouldIgnore(relPosix(dir, resolve(dir, r.file)), cfg)) return false;
     return true;
   });
 }
@@ -118,8 +120,15 @@ function scanNextJsRoutes(dir) {
       const relDir = relative(resolve(dir, apiBase), dirname(filePath));
       const apiPath = '/' + relDir
         .replace(/\\/g, '/')
-        .replace(/\[\.\.\.(\w+)\]/g, ':$1*')  // Catch-all [...slug]
-        .replace(/\[(\w+)\]/g, ':$1');          // Dynamic [id]
+        // Strip route-group segments like `(admin)` — they organize files but
+        // do NOT appear in the URL. The frontend scanner already does this; the
+        // route scanner used to leak them, e.g. `/api/(admin)/users`.
+        .split('/')
+        .filter(seg => seg && !/^\(.*\)$/.test(seg))
+        .join('/')
+        .replace(/\[\[\.\.\.(\w+)\]\]/g, ':$1*')  // Optional catch-all [[...slug]] — before [...slug]
+        .replace(/\[\.\.\.(\w+)\]/g, ':$1*')        // Catch-all [...slug]
+        .replace(/\[(\w+)\]/g, ':$1');               // Dynamic [id]
 
       // Extract exported HTTP methods
       const methods = ['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'HEAD', 'OPTIONS'];
@@ -195,9 +204,22 @@ function scanNextJsRoutes(dir) {
 // ── Express / Generic Node.js ───────────────────────────────────────────────
 
 function scanExpressRoutes(dir, roots = null) {
-  const routes = [];
+  // Regex is the FALLBACK (used only when @babel/parser can't parse a file).
+  // It hardcodes app/router/server receivers; the AST path matches any receiver.
   const routePattern = /(?:app|router|server)\s*\.\s*(get|post|put|delete|patch|head|options)\s*\(\s*['"`]([^'"`]+)['"`]/gi;
 
+  // ── Phase 0: collect every candidate file ONCE ──────────────────────────────
+  // The mount map (phase 1) and the route emit (phase 2) must see the same set,
+  // and we don't want to walk the tree twice.
+  const files = [];                 // { content, filePath, fileLabel }
+  const seenPaths = new Set();
+  const addFile = (filePath, fileLabel) => {
+    if (seenPaths.has(filePath)) return;
+    const content = readFileSafe(filePath);
+    if (!content) return;
+    seenPaths.add(filePath);
+    files.push({ content, filePath, fileLabel });
+  };
   // Monorepo-aware: walk resolved absolute source roots when provided,
   // otherwise fall back to conventional root-relative directories.
   const searchTargets = roots && roots.length
@@ -205,51 +227,114 @@ function scanExpressRoutes(dir, roots = null) {
     : ['src', 'routes', 'api', 'server', 'lib'].map(d => resolve(dir, d));
   for (const fullDir of searchTargets) {
     if (!existsSync(fullDir)) continue;
-
     walkRouteDirs(fullDir, (filePath) => {
-      if (!isJSFile(filePath)) return;
-      const content = readFileSafe(filePath);
-      if (!content) return;
+      if (isJSFile(filePath)) addFile(filePath, relative(dir, filePath));
+    });
+  }
+  for (const rootFile of ['app.js', 'app.mjs', 'app.ts', 'server.js', 'server.ts', 'index.js', 'index.ts']) {
+    const filePath = resolve(dir, rootFile);
+    if (existsSync(filePath)) addFile(filePath, rootFile);
+  }
 
-      let match;
-      const regex = new RegExp(routePattern.source, 'gi');
-      while ((match = regex.exec(content)) !== null) {
+  // ── Phase 1: build the mount map ────────────────────────────────────────────
+  // absFilePath -> [{ receiver|null, prefix }].  `receiver === null` means the
+  // prefix applies to EVERY route in that file (an imported sub-router); a
+  // non-null receiver means it applies only to routes whose receiver matches
+  // (a same-file `const r = Router(); app.use('/api', r)` — so a sibling
+  // `app.get('/health')` in the same file is NOT wrongly prefixed).
+  const mountMap = buildExpressMountMap(files);
+
+  // ── Phase 2: emit routes, prefixing by mount where known ────────────────────
+  const routes = [];
+  for (const { content, filePath, fileLabel } of files) {
+    const mounts = mountMap.get(filePath) || [];
+    const emit = (method, path, index, receiver) => {
+      const prefixes = mounts
+        .filter(m => m.receiver === null || m.receiver === receiver)
+        .map(m => m.prefix);
+      const finalPaths = prefixes.length ? prefixes.map(p => joinRoutePath(p, path)) : [path];
+      for (const fullPath of finalPaths) {
         routes.push({
-          method: match[1].toUpperCase(),
-          path: match[2],
-          handler: extractHandlerName(content, match.index),
-          file: relative(dir, filePath),
+          method: method.toUpperCase(),
+          path: fullPath,
+          handler: extractHandlerName(content, index),
+          file: fileLabel,
           source: 'express',
-          auth: hasAuthMiddleware(content, match[2]),
-          description: extractNearbyComment(content, match.index),
+          auth: hasAuthMiddleware(content, path),
+          description: extractNearbyComment(content, index),
         });
       }
-    });
+    };
+    const ast = extractJsRouteCalls(content, filePath);
+    if (ast) {
+      for (const r of ast) emit(r.method, r.path, r.start, r.receiver ?? null);
+    } else {
+      const regex = new RegExp(routePattern.source, 'gi');
+      let match;
+      while ((match = regex.exec(content)) !== null) emit(match[1], match[2], match.index, null);
+    }
   }
 
-  // Also check root files
-  for (const rootFile of ['app.js', 'app.mjs', 'app.ts', 'server.js', 'server.ts', 'index.js', 'index.ts']) {
-    const filePath = resolve(dir, rootFile);
-    if (!existsSync(filePath)) continue;
-    const content = readFileSafe(filePath);
-    if (!content) return;
+  return routes;
+}
 
-    let match;
-    const regex = new RegExp(routePattern.source, 'gi');
-    while ((match = regex.exec(content)) !== null) {
-      routes.push({
-        method: match[1].toUpperCase(),
-        path: match[2],
-        handler: extractHandlerName(content, match.index),
-        file: rootFile,
-        source: 'express',
-        auth: hasAuthMiddleware(content, match[2]),
-        description: extractNearbyComment(content, match.index),
-      });
+/**
+ * Build the Express mount map from the collected files (phase 1 above).
+ * For each `<x>.use('/prefix', router)`:
+ *   - router is an IMPORTED binding  → the prefix applies to ALL routes in the
+ *     resolved target file (receiver: null). One router per file is the norm.
+ *   - router is a LOCAL identifier    → the prefix applies only to that file's
+ *     routes whose receiver matches (receiver: ident).
+ *
+ * Known limitations (documented, not silently wrong): transitive composition
+ * (`app.use('/api', api)` then `api.use('/x', x)` does NOT yield `/api/x`), and
+ * dynamic mount paths (non-string-literal prefixes) are skipped. Unmounted
+ * files keep their bare paths — exactly the pre-mount-map behavior.
+ */
+function buildExpressMountMap(files) {
+  const map = new Map();
+  const add = (absFile, receiver, prefix) => {
+    if (!map.has(absFile)) map.set(absFile, []);
+    map.get(absFile).push({ receiver, prefix });
+  };
+  for (const { content, filePath } of files) {
+    const mi = extractJsMountsAndImports(content, filePath);
+    if (!mi) continue;
+    for (const { prefix, ident } of mi.mounts) {
+      const spec = mi.imports[ident];
+      if (spec) {
+        const target = resolveLocalImport(filePath, spec);
+        if (target) add(target, null, prefix);
+      } else {
+        add(filePath, ident, prefix);
+      }
     }
   }
+  return map;
+}
 
-  return routes;
+/** Resolve a RELATIVE import specifier to an absolute file path (best effort). */
+function resolveLocalImport(fromFile, spec) {
+  if (!spec.startsWith('.')) return null; // bare/node_modules specifiers aren't our routers
+  const base = resolve(dirname(fromFile), spec);
+  for (const ext of ['', '.ts', '.js', '.mjs', '.cjs', '.tsx', '.jsx']) {
+    const cand = base + ext;
+    try { if (existsSync(cand) && statSync(cand).isFile()) return cand; } catch { /* skip */ }
+  }
+  for (const idx of ['index.ts', 'index.js', 'index.mjs']) {
+    const cand = join(base, idx);
+    try { if (existsSync(cand) && statSync(cand).isFile()) return cand; } catch { /* skip */ }
+  }
+  return null;
+}
+
+/** Join a mount prefix and a route path into one normalized `/a/b` path. */
+function joinRoutePath(prefix, p) {
+  if (!prefix) return p;
+  const left = prefix.replace(/\/+$/, '');        // drop trailing slash(es)
+  const right = (p === '/' || p === '') ? '' : ('/' + p.replace(/^\/+/, '')); // single leading slash
+  const joined = left + right;
+  return joined || '/';
 }
 
 // ── Fastify ─────────────────────────────────────────────────────────────────
@@ -266,18 +351,28 @@ function scanFastifyRoutes(dir, roots = null) {
       const content = readFileSafe(filePath);
       if (!content) return;
 
-      let match;
-      const regex = new RegExp(pattern.source, 'gi');
-      while ((match = regex.exec(content)) !== null) {
-        routes.push({
-          method: match[1].toUpperCase(),
-          path: match[2],
-          handler: extractHandlerName(content, match.index),
-          file: relative(dir, filePath),
-          source: 'fastify',
-          auth: hasAuthCheck(content),
-          description: extractNearbyComment(content, match.index),
-        });
+      const emit = (method, path, index) => routes.push({
+        method: method.toUpperCase(),
+        path,
+        handler: extractHandlerName(content, index),
+        file: relative(dir, filePath),
+        source: 'fastify',
+        auth: hasAuthCheck(content),
+        description: extractNearbyComment(content, index),
+      });
+
+      // AST-first: method shorthand (fastify.get('/x')) AND the declarative
+      // object form (fastify.route({ method, url })) the regex never matched.
+      // Both return null only on parse failure → regex fallback.
+      const calls = extractJsRouteCalls(content, filePath);
+      const objs = extractJsRouteObjects(content, filePath);
+      if (calls || objs) {
+        for (const r of calls || []) emit(r.method, r.path, r.start);
+        for (const r of objs || []) emit(r.method, r.path, r.start);
+      } else {
+        let match;
+        const regex = new RegExp(pattern.source, 'gi');
+        while ((match = regex.exec(content)) !== null) emit(match[1], match[2], match.index);
       }
     });
   }
@@ -302,18 +397,25 @@ function scanHonoRoutes(dir, roots = null) {
       const content = readFileSafe(filePath);
       if (!content) return;
 
-      let match;
-      const regex = new RegExp(pattern.source, 'gi');
-      while ((match = regex.exec(content)) !== null) {
-        routes.push({
-          method: match[1].toUpperCase(),
-          path: match[2],
-          handler: '',
-          file: relative(dir, filePath),
-          source: 'hono',
-          auth: hasAuthCheck(content),
-          description: extractNearbyComment(content, match.index),
-        });
+      const emit = (method, path, index) => routes.push({
+        method: method.toUpperCase(),
+        path,
+        handler: '',
+        file: relative(dir, filePath),
+        source: 'hono',
+        auth: hasAuthCheck(content),
+        description: extractNearbyComment(content, index),
+      });
+
+      // AST-first (any receiver, multi-line, template paths — Hono/Koa method
+      // shorthand `app.get('/x')` / `router.get('/x')`); regex fallback.
+      const calls = extractJsRouteCalls(content, filePath);
+      if (calls) {
+        for (const r of calls) emit(r.method, r.path, r.start);
+      } else {
+        let match;
+        const regex = new RegExp(pattern.source, 'gi');
+        while ((match = regex.exec(content)) !== null) emit(match[1], match[2], match.index);
       }
     });
   }
@@ -357,10 +459,33 @@ function scanFastAPIRoutes(dir) {
   const pattern = /@(?:app|router)\s*\.\s*(get|post|put|delete|patch)\s*\(\s*['"]([^'"]+)['"]/gi;
 
   const pyFiles = findFiles(dir, /\.py$/);
+  // AST-first: ONE python3 subprocess parses every file. `null` means Python is
+  // unavailable or the subprocess failed → regex fallback for all files. A
+  // per-file `ok:false` falls back for just that file. The AST form also reads
+  // multi-line decorators and Flask `methods=[...]` arrays the regex misses.
+  const astByFile = extractPythonFiles(pyFiles);
+
   for (const filePath of pyFiles) {
     const content = readFileSafe(filePath);
     if (!content) continue;
 
+    const parsed = astByFile && astByFile[filePath];
+    const fileAuth = content.includes('Depends(') && content.includes('auth');
+    if (parsed && parsed.ok) {
+      for (const r of parsed.routes || []) {
+        routes.push({
+          method: r.method,
+          path: r.path,
+          handler: r.func || '',
+          file: relative(dir, filePath),
+          source: 'fastapi',
+          auth: fileAuth,
+          description: r.desc || '',
+        });
+      }
+      continue;
+    }
+
     let match;
     const regex = new RegExp(pattern.source, 'gi');
     while ((match = regex.exec(content)) !== null) {
@@ -370,7 +495,7 @@ function scanFastAPIRoutes(dir) {
         handler: extractPythonFunctionName(content, match.index),
         file: relative(dir, filePath),
         source: 'fastapi',
-        auth: content.includes('Depends(') && content.includes('auth'),
+        auth: fileAuth,
         description: extractPythonDocstring(content, match.index),
       });
     }
@@ -515,7 +640,7 @@ function scanRustWebRoutes(dir) {
 // ── Helpers ──────────────────────────────────────────────────────────────────
 
 function readFileSafe(path) {
-  try { return readFileSync(path, 'utf-8'); } catch { return null; }
+  return readScannable(path); // size-capped; skips minified/generated bundles
 }
 
 function isJSFile(path) {