botversion-sdk 2.2.2 → 2.3.0

package/scanner-sdk/scan/express-static.js

@@ -0,0 +1,292 @@
+// ─────────────────────────────────────────────────────────────
+// FILE: scan/express-static.js
+// PURPOSE: Reads Express routes written in files (including .route() chains, constant paths and arrays of paths).
+// CONTAINS: findNonLiteralRoutes, namedHandlerCodeFromCall, scanExpressFileStatically
+// USED BY: scan/express.js
+// DEPENDS ON: ./text-utils, ./field-extractors, ./route-segments, ./response-shape
+// ─────────────────────────────────────────────────────────────
+"use strict";
+
+const {
+  importedHandlerCode,
+  blankOutComments,
+  buildRouteParamMap,
+  collectStringConstants,
+  joinMountPath,
+  resolveRouteTemplate,
+  sliceCallText,
+  splitStaticEntries,
+} = require("./text-utils");
+const { buildLocalMountMap } = require("./express-files");
+const {
+  extractBodyFieldsForRoute,
+  extractBodyFieldsFromFile,
+  extractMultipartFieldsFromFile,
+  extractQueryFieldsFromFile,
+  findNamedHandlerCode,
+  mergeBodyWithMultipart,
+} = require("./field-extractors");
+const { sliceRouteSegment } = require("./route-segments");
+const {
+  extractResponseFieldsForRoute,
+  extractResponseFieldsFromSegment,
+} = require("./response-shape");
+
+// Finds routes whose path is not a plain quoted text: .route("/x").get(...) chains,
+// paths stored in constants, and arrays of paths. Returns { method, path, own } items,
+// where "own" is the exact code of that one call.
+function findNonLiteralRoutes(content, consts, literalPatternSource) {
+  const found = [];
+  const literalAt = new Set();
+  for (const mm of content.matchAll(new RegExp(literalPatternSource, "gi"))) {
+    literalAt.add(mm.index);
+  }
+  const chainAt = new Set();
+
+  function pathsFromArg(arg) {
+    if (!arg) return [];
+    const list = /^\[([\s\S]*)\]$/.exec(arg);
+    const parts = list ? splitStaticEntries(list[1]) || [] : [arg];
+    const out = [];
+    parts.forEach(function (part) {
+      let value = null;
+      const quoted = /^(['"`])([\s\S]*)\1$/.exec(part);
+      if (quoted) value = resolveRouteTemplate(quoted[2], consts);
+      else if (Object.prototype.hasOwnProperty.call(consts, part))
+        value = consts[part];
+      if (typeof value === "string" && value.charAt(0) === "/") out.push(value);
+    });
+    return out;
+  }
+
+  // router.route("/x").get(...).post(...)
+  for (const rm of content.matchAll(/\.route\s*\(\s*([^)]*?)\s*\)/g)) {
+    const paths = pathsFromArg(rm[1].trim());
+    let pos = rm.index + rm[0].length;
+    for (;;) {
+      const cm = /^\s*\.\s*(get|post|put|delete|patch|all)\s*\(/i.exec(
+        content.slice(pos, pos + 60),
+      );
+      if (!cm) break;
+      const dotIndex = pos + cm[0].indexOf(".");
+      const text = sliceCallText(content, dotIndex);
+      if (!text) break;
+      chainAt.add(dotIndex);
+      paths.forEach(function (p) {
+        found.push({ method: cm[1].toUpperCase(), path: p, own: text });
+      });
+      pos = dotIndex + text.length;
+    }
+  }
+
+  // router.get(PATH_CONSTANT, ...) and router.post(["/a", "/b"], ...)
+  for (const cm of content.matchAll(
+    /\.(get|post|put|delete|patch|all)\s*\(/gi,
+  )) {
+    if (literalAt.has(cm.index) || chainAt.has(cm.index)) continue;
+    const text = sliceCallText(content, cm.index);
+    if (!text) continue;
+    const args = splitStaticEntries(
+      text.slice(text.indexOf("(") + 1, text.length - 1),
+    );
+    if (!args || args.length === 0) continue;
+    pathsFromArg(args[0]).forEach(function (p) {
+      found.push({ method: cm[1].toUpperCase(), path: p, own: text });
+    });
+  }
+  return found;
+}
+
+// Code of handlers that are written elsewhere in the same file and passed by name,
+// e.g. .post(requireAuth, createProject) or .post(wrap(createProject))
+function namedHandlerCodeFromCall(callText, content) {
+  const args =
+    splitStaticEntries(
+      callText.slice(callText.indexOf("(") + 1, callText.length - 1),
+    ) || [];
+  let code = "";
+  args.forEach(function (arg) {
+    const wrapped = /^[\w$.]+\s*\(\s*([A-Za-z_$][\w$]*)\s*\)$/.exec(arg);
+    const name = wrapped
+      ? wrapped[1]
+      : /^[A-Za-z_$][\w$]*$/.test(arg)
+        ? arg
+        : null;
+    if (!name) return;
+    const found = findNamedHandlerCode(content, name);
+    if (found) code += "\n" + found;
+  });
+  return code;
+}
+
+function scanExpressFileStatically(filePath, seen) {
+  const fs = require("fs");
+  const endpoints = [];
+
+  let content;
+  try {
+    content = fs.readFileSync(filePath, "utf8");
+  } catch {
+    return endpoints;
+  }
+  // Comments are blanked out so commented-out routes are never reported
+  content = blankOutComments(content);
+  const routeConsts = collectStringConstants(content);
+  const localMounts = buildLocalMountMap(content);
+
+  // Match patterns like:
+  // app.get('/path', ...)
+  // router.post('/path', ...)
+  // app.all('/curd.php', ...)
+  const routePattern =
+    /\.(get|post|put|delete|patch|all)\s*\(\s*['"`]([^'"`]+)['"`]/gi;
+
+  const debugMatchedAt = new Set();
+  for (const mm of content.matchAll(new RegExp(routePattern.source, "gi"))) {
+    debugMatchedAt.add(mm.index);
+  }
+  seen._debugFrom = seen._debugFrom || {};
+
+  let match;
+  while ((match = routePattern.exec(content)) !== null) {
+    const method = match[1].toUpperCase();
+    // Paths written as `/${CONSTANT}` are filled in from constants in the same file
+    const resolvedPath = resolveRouteTemplate(match[2], routeConsts);
+    // A router mounted in this same file (app.use("/api", router)) carries its prefix with it
+    const receiver = /([A-Za-z_$][\w$]*)\s*$/.exec(
+      content.slice(Math.max(0, match.index - 80), match.index),
+    );
+    const localPrefix =
+      receiver && localMounts[receiver[1]] ? localMounts[receiver[1]] : "";
+    const routePath = localPrefix
+      ? joinMountPath(
+          localPrefix,
+          resolvedPath === null ? match[2] : resolvedPath,
+        )
+      : resolvedPath === null
+        ? match[2]
+        : resolvedPath;
+    if (resolvedPath === null) {
+      continue;
+    }
+
+    const debugLineStart = content.lastIndexOf("\n", match.index) + 1;
+    const debugLineBefore = content.slice(debugLineStart, match.index);
+
+    // Skip middleware patterns
+    if (routePath.includes("*")) {
+      continue;
+    }
+
+    const key = method + ":" + routePath;
+    if (seen.has(key)) {
+      continue;
+    }
+    seen.add(key);
+    seen._debugFrom[key] = filePath;
+
+    const effectiveMethod = method === "ALL" ? "GET" : method;
+    const needsBody = ["POST", "PUT", "PATCH"].includes(effectiveMethod);
+    // Handler imported from another file (e.g. controllers/x.js): its code is read too
+    const importedCode = importedHandlerCode(
+      content,
+      sliceCallText(content, match.index),
+      filePath,
+    );
+    // Read only THIS route's own code, so fields from other routes in the file are never mixed in
+    const bodyFields = needsBody
+      ? extractBodyFieldsForRoute(content, match.index, routePattern.source) ||
+        (importedCode ? extractBodyFieldsFromFile(importedCode) : null)
+      : null;
+
+    // GET inputs come from the query string; read them from this route's own code
+    const queryFields =
+      effectiveMethod === "GET"
+        ? extractQueryFieldsFromFile(
+            sliceRouteSegment(content, match.index, routePattern.source),
+          ) || (importedCode ? extractQueryFieldsFromFile(importedCode) : null)
+        : null;
+
+    const routeParamMap = buildRouteParamMap(routePath, []);
+
+    const debugSegment = sliceRouteSegment(
+      content,
+      match.index,
+      routePattern.source,
+    );
+
+    endpoints.push({
+      method: effectiveMethod,
+      path: routePath,
+      description: "",
+      requestBody:
+        mergeBodyWithMultipart(
+          bodyFields,
+          ["POST", "PUT", "PATCH"].includes(effectiveMethod)
+            ? extractMultipartFieldsFromFile(
+                sliceRouteSegment(content, match.index, routePattern.source),
+              ) ||
+                ((content.match(new RegExp(routePattern.source, "gi")) || [])
+                  .length === 1
+                  ? extractMultipartFieldsFromFile(content)
+                  : null)
+            : null,
+        ) || queryFields,
+      routeParamMap: routeParamMap,
+      responseBody:
+        extractResponseFieldsForRoute(
+          content,
+          match.index,
+          routePattern.source,
+        ) ||
+        (importedCode ? extractResponseFieldsFromSegment(importedCode) : null),
+      detectedBy: "static-scan-file",
+    });
+  }
+
+  // Routes whose path is not a plain quoted text: .route("/x").get(...) chains, constant paths, arrays of paths
+  const extraSeen = new Set(
+    endpoints.map(function (e) {
+      return e.method + ":" + e.path;
+    }),
+  );
+  findNonLiteralRoutes(content, routeConsts, routePattern.source).forEach(
+    function (r) {
+      const effectiveMethod = r.method === "ALL" ? "GET" : r.method;
+      const extraKey = effectiveMethod + ":" + r.path;
+      if (extraSeen.has(extraKey)) {
+        return;
+      }
+      extraSeen.add(extraKey);
+      const allCode =
+        r.own +
+        namedHandlerCodeFromCall(r.own, content) +
+        importedHandlerCode(content, r.own, filePath);
+      const needsBody = ["POST", "PUT", "PATCH"].includes(effectiveMethod);
+      const bodyFields = needsBody ? extractBodyFieldsFromFile(allCode) : null;
+      const multipartFields = needsBody
+        ? extractMultipartFieldsFromFile(allCode)
+        : null;
+      const queryFields =
+        effectiveMethod === "GET" || effectiveMethod === "DELETE"
+          ? extractQueryFieldsFromFile(allCode)
+          : null;
+      endpoints.push({
+        method: effectiveMethod,
+        path: r.path,
+        description: "",
+        requestBody:
+          mergeBodyWithMultipart(bodyFields, multipartFields) || queryFields,
+        routeParamMap: buildRouteParamMap(r.path, []),
+        responseBody: extractResponseFieldsFromSegment(allCode),
+        detectedBy: "static-scan-file",
+      });
+    },
+  );
+
+  return endpoints;
+}
+
+module.exports = {
+  scanExpressFileStatically,
+};

package/scanner-sdk/scan/express.js

@@ -0,0 +1,217 @@
+// ─────────────────────────────────────────────────────────────
+// FILE: scan/express.js
+// PURPOSE: Main Express scan: reads the live Express app, then adds routes found by reading files.
+// CONTAINS: BODYDBG, bodyKeys, scanExpressRoutes, extractRoutes, extractHandlerName
+// USED BY: scanner.js (scanExpressRoutes)
+// DEPENDS ON: ./text-utils, ./express-files, ./express-static, ./field-extractors, ./response-shape
+// ─────────────────────────────────────────────────────────────
+"use strict";
+
+const {
+  buildRouteParamMap,
+  joinMountPath,
+  regexpToPath,
+} = require("./text-utils");
+const {
+  buildBodyMap,
+  buildMountPrefixMap,
+  scanAllExpressFiles,
+} = require("./express-files");
+const { scanExpressFileStatically } = require("./express-static");
+const {
+  extractBodyFieldsFromFile,
+  extractQueryFieldsFromHandlers,
+} = require("./field-extractors");
+const { extractResponseFieldsFromHandlers } = require("./response-shape");
+
+function bodyKeys(schema) {
+  return schema && schema.properties ? Object.keys(schema.properties) : null;
+}
+
+/**
+ * Scans an Express app object and extracts all registered routes
+ */
+function scanExpressRoutes(app, cwd) {
+  const endpoints = [];
+  const seen = new Set();
+  try {
+    // Reads a file by runtime path without require() so bundlers don't warn about dynamic requires
+    const expressPkgPath = require("path").join(
+      cwd,
+      "node_modules",
+      "express",
+      "package.json",
+    );
+    const expressPkg = JSON.parse(
+      require("fs").readFileSync(expressPkgPath, "utf8"),
+    );
+  } catch (e) {}
+
+  // Build body map from ALL files upfront
+  const bodyMap = cwd ? buildBodyMap(cwd) : {};
+
+  if (app) {
+    if (app.lazyrouter) app.lazyrouter();
+    const router = app._router || app.router || (app.stack ? app : null);
+    if (router) {
+      const stack = router.stack || [];
+      try {
+        extractRoutes(stack, "", endpoints, seen, bodyMap);
+      } catch (e) {
+        throw e;
+      }
+    }
+  }
+
+  // ALSO scan all JS/TS files statically for route definitions
+  if (cwd) {
+    const allFiles = scanAllExpressFiles(cwd);
+    // Router files only know their own relative paths, so add the prefix each one is mounted under
+    const mountPrefixes = buildMountPrefixMap(allFiles);
+    for (const file of allFiles) {
+      const fileEndpoints = scanExpressFileStatically(file, new Set());
+      const prefixes =
+        mountPrefixes[file] && mountPrefixes[file].length > 0
+          ? mountPrefixes[file]
+          : [""];
+      fileEndpoints.forEach(function (ep) {
+        prefixes.forEach(function (prefix) {
+          const fullPath = joinMountPath(prefix, ep.path);
+          const key = ep.method + ":" + fullPath;
+          if (seen.has(key) || seen.has(key + "/")) {
+            return;
+          }
+          seen.add(key);
+          endpoints.push(
+            Object.assign({}, ep, {
+              path: fullPath,
+              routeParamMap: buildRouteParamMap(fullPath, []),
+            }),
+          );
+        });
+      });
+    }
+  }
+
+  return endpoints;
+}
+
+/**
+ * Recursively walks Express router stack and pulls out route layers
+ *
+ * Each layer in the stack is one of:
+ *   - a route layer  → layer.route exists, has .path and .methods
+ *   - a router layer → layer.name === 'router', has its own .handle.stack
+ *   - middleware     → everything else (body-parser, cors, etc.) — skip these
+ */
+function extractRoutes(stack, prefix, endpoints, seen, bodyMap) {
+  prefix = prefix || "";
+
+  stack.forEach(function (layer) {
+    // ── Route layer (app.get / app.post / etc.) ──────────────────────────
+    if (layer.route) {
+      var routePath = prefix + (layer.route.path || "");
+      var methods = Object.keys(layer.route.methods).filter(function (m) {
+        return layer.route.methods[m] === true;
+      });
+
+      methods.forEach(function (method) {
+        method = method.toUpperCase();
+        if (method === "_ALL") return;
+
+        var key = method + ":" + routePath;
+        if (seen.has(key)) return;
+        seen.add(key);
+
+        const needsBody = ["POST", "PUT", "PATCH"].includes(method);
+        let requestBody = null;
+
+        if (needsBody) {
+          // Strategy 1: scan inline handler directly via fn.toString() — most accurate
+          for (const handler of layer.route.stack) {
+            const fn = handler.handle || handler;
+            const fnStr = fn.toString();
+            const fields = extractBodyFieldsFromFile(fnStr);
+            if (fields) {
+              requestBody = fields;
+              break;
+            }
+          }
+
+          // Strategy 2: fall back to bodyMap using handler name
+          if (!requestBody) {
+            const handlerName = extractHandlerName(layer);
+            if (handlerName && bodyMap[handlerName]) {
+              requestBody = bodyMap[handlerName];
+            }
+          }
+        }
+
+        // GET inputs come from the query string, so read them from the handler code
+        if (method === "GET") {
+          requestBody = extractQueryFieldsFromHandlers(layer.route.stack);
+        }
+
+        const routeParamMap = buildRouteParamMap(routePath, []);
+
+        endpoints.push({
+          method: method,
+          path: routePath,
+          description: "",
+          requestBody: requestBody,
+          responseBody: extractResponseFieldsFromHandlers(
+            layer.route.stack,
+            method,
+          ),
+          routeParamMap: routeParamMap,
+          detectedBy: "static-scan",
+        });
+      });
+      return;
+    }
+
+    // ── Nested router layer (app.use('/prefix', router)) ─────────────────
+    if (layer.name === "router" && layer.handle && layer.handle.stack) {
+      // Extract the mount path from the regexp
+      var mountPath = prefix + regexpToPath(layer.regexp, layer.keys);
+      extractRoutes(layer.handle.stack, mountPath, endpoints, seen, bodyMap);
+      return;
+    }
+
+    // ── Everything else is middleware — skip ─────────────────────────────
+  });
+}
+
+function extractHandlerName(layer) {
+  const handlers = layer.route.stack.map((h) => h.handle || h);
+
+  const SKIP_NAMES = new Set([
+    "anonymous",
+    "",
+    "bound dispatch",
+    "middleware",
+    "protect",
+    "admin",
+    "auth",
+    "verify",
+    "validate",
+    "isAuth",
+    "isAdmin",
+    "checkAuth",
+    "authenticate",
+  ]);
+
+  // Try from last to first, skip known middleware names
+  for (let i = handlers.length - 1; i >= 0; i--) {
+    const fn = handlers[i];
+    const name = fn.name || "";
+    if (name && !SKIP_NAMES.has(name) && !name.startsWith("bound ")) {
+      return name;
+    }
+  }
+  return null;
+}
+
+module.exports = {
+  scanExpressRoutes,
+};