carto-md 1.0.0

package/src/extractors/dbtables.js

@@ -0,0 +1,80 @@
+/**
+ * extractDBTables(content) → Array<{ tableName, modelName }>
+ *
+ * Supports two ORM patterns:
+ *   1. SQLAlchemy: class Foo(Base): __tablename__ = 'foo'
+ *   2. Django ORM: class Foo(models.Model): class Meta: db_table = 'foo'
+ *      (falls back to snake_case of class name if no db_table)
+ */
+function extractDBTables(content) {
+  const tables = [];
+  const lines = content.split('\n');
+
+  // Pattern 1 — SQLAlchemy
+  const sqlaClassPattern = /^class\s+(\w+)\s*\(.*Base.*\)\s*:/gm;
+  let classMatch;
+  while ((classMatch = sqlaClassPattern.exec(content)) !== null) {
+    const modelName = classMatch[1];
+    const classEnd = classMatch.index + classMatch[0].length;
+
+    // Find line index of this class
+    const textBefore = content.substring(0, classEnd);
+    const lineIndex = textBefore.split('\n').length - 1;
+
+    // Look ahead up to 10 lines for __tablename__
+    let found = false;
+    for (let j = lineIndex + 1; j < Math.min(lineIndex + 11, lines.length); j++) {
+      const tnMatch = lines[j].match(/^\s+__tablename__\s*=\s*['"]([^'"]+)['"]/);
+      if (tnMatch) {
+        tables.push({ tableName: tnMatch[1], modelName });
+        found = true;
+        break;
+      }
+      // Stop if we hit another class definition
+      if (/^class\s/.test(lines[j])) break;
+    }
+    // If not found, skip — not a table-mapped model
+  }
+
+  // Pattern 2 — Django ORM
+  const djangoClassPattern = /^class\s+(\w+)\s*\(.*models\.Model.*\)\s*:/gm;
+  while ((classMatch = djangoClassPattern.exec(content)) !== null) {
+    const modelName = classMatch[1];
+    const classEnd = classMatch.index + classMatch[0].length;
+
+    const textBefore = content.substring(0, classEnd);
+    const lineIndex = textBefore.split('\n').length - 1;
+
+    // Look ahead up to 15 lines for db_table
+    let found = false;
+    for (let j = lineIndex + 1; j < Math.min(lineIndex + 16, lines.length); j++) {
+      const dtMatch = lines[j].match(/db_table\s*=\s*['"]([^'"]+)['"]/);
+      if (dtMatch) {
+        tables.push({ tableName: dtMatch[1], modelName });
+        found = true;
+        break;
+      }
+      if (/^class\s/.test(lines[j])) break;
+    }
+
+    // Django default: snake_case of class name
+    if (!found) {
+      tables.push({ tableName: toSnakeCase(modelName), modelName });
+    }
+  }
+
+  return tables;
+}
+
+/**
+ * Convert PascalCase to snake_case.
+ * MyModel → my_model
+ */
+function toSnakeCase(str) {
+  return str
+    .replace(/([A-Z])/g, '_$1')
+    .toLowerCase()
+    .replace(/^_/, '');
+}
+
+module.exports = { extractDBTables };

package/src/extractors/envvars.js

@@ -0,0 +1,40 @@
+/**
+ * extractEnvVars(content) → Array<string>
+ *
+ * Extracts environment variable NAMES ONLY. Never values. Never defaults.
+ * Returns deduplicated, alphabetically sorted array.
+ *
+ * Supported patterns (Python only):
+ *   os.getenv('VAR')
+ *   os.getenv("VAR")
+ *   os.environ.get('VAR')
+ *   os.environ.get("VAR")
+ *   os.environ['VAR']
+ *   os.environ["VAR"]
+ */
+function extractEnvVars(content) {
+  const vars = new Set();
+
+  // os.getenv('VAR') or os.getenv("VAR")
+  const getenvPattern = /os\.getenv\s*\(\s*['"]([^'"]+)['"]/g;
+  let match;
+  while ((match = getenvPattern.exec(content)) !== null) {
+    vars.add(match[1]);
+  }
+
+  // os.environ.get('VAR') or os.environ.get("VAR")
+  const environGetPattern = /os\.environ\.get\s*\(\s*['"]([^'"]+)['"]/g;
+  while ((match = environGetPattern.exec(content)) !== null) {
+    vars.add(match[1]);
+  }
+
+  // os.environ['VAR'] or os.environ["VAR"]
+  const environBracketPattern = /os\.environ\s*\[\s*['"]([^'"]+)['"]\s*\]/g;
+  while ((match = environBracketPattern.exec(content)) !== null) {
+    vars.add(match[1]);
+  }
+
+  return [...vars].sort();
+}
+
+module.exports = { extractEnvVars };

package/src/extractors/filemap.js

@@ -0,0 +1,36 @@
+const path = require('path');
+
+/**
+ * inferResponsibility(filename, functionCount, routeCount) → string
+ *
+ * Returns a short description of what a file does.
+ * If nothing matches confidently → return "—".
+ */
+function inferResponsibility(filename, functionCount, routeCount) {
+  const base = path.basename(filename).toLowerCase();
+
+  // Skip __init__.py entirely
+  if (base === '__init__.py') return null;
+
+  // Check in order, return first match
+  if (routeCount > 0) return `API routes (${routeCount} routes)`;
+  if (base.includes('collect')) return 'Data collection';
+  if (base.includes('rule') || base.includes('check')) return `Rule checks (${functionCount} functions)`;
+  if (base.includes('score') || base.includes('scoring')) return 'Scoring logic';
+  if (base.includes('llm') || base.includes('ai') || base.includes('ml')) return 'AI/LLM integration';
+  if (base.includes('github')) return 'GitHub integration';
+  if (base.includes('storage')) return 'Storage operations';
+  if (base.includes('drift')) return 'Drift detection';
+  if (base.includes('auth')) return 'Authentication';
+  if (base.includes('database') || base.includes('db')) return 'Database operations';
+  if (base.includes('model')) return 'Data models';
+  if (base.includes('test') || base.includes('spec')) return 'Tests';
+  if (base.includes('util') || base.includes('helper')) return 'Utilities';
+  if (base.includes('config') || base.includes('setting')) return 'Configuration';
+  if (base.includes('middleware')) return 'Middleware';
+  if (functionCount > 0) return `${functionCount} functions`;
+
+  return '\u2014';
+}
+
+module.exports = { inferResponsibility };

package/src/extractors/frontend.js

@@ -0,0 +1,43 @@
+/**
+ * Extracts fetch() calls and sessionStorage usage from HTML/JS content.
+ * Strips newlines before fetch matching to handle multiline fetch options.
+ * Also detects dynamic fetch calls where the URL is a variable.
+ */
+function extractFrontend(content) {
+  const fetches = [];
+  const storageKeys = [];
+
+  // Strip newlines so [^}]* can cross what were originally separate lines
+  const singleLineContent = content.replace(/\n/g, ' ');
+
+  const fetchPattern = /fetch\s*\(\s*[`"']([^`"']+)[`"']\s*(?:,\s*\{[^}]*method\s*:\s*["'](\w+)["'][^}]*\})?/g;
+  let match;
+  while ((match = fetchPattern.exec(singleLineContent)) !== null) {
+    fetches.push({
+      url: match[1],
+      method: match[2] ? match[2].toUpperCase() : 'GET'
+    });
+  }
+
+  // Detect dynamic fetch calls — fetch(variable, ...) where variable is a bare identifier
+  const dynamicFetchPattern = /fetch\s*\(\s*([a-zA-Z_$][a-zA-Z0-9_$]*)\s*[,)]/g;
+  const reserved = new Set(['true', 'false', 'null', 'undefined']);
+  while ((match = dynamicFetchPattern.exec(singleLineContent)) !== null) {
+    if (!reserved.has(match[1])) {
+      fetches.push({ url: '[dynamic]', method: '[dynamic]' });
+    }
+  }
+
+  // sessionStorage — original content is fine, these are single-line
+  const storagePattern = /sessionStorage\.(getItem|setItem)\s*\(\s*["']([^"']+)["']/g;
+  while ((match = storagePattern.exec(content)) !== null) {
+    storageKeys.push({
+      operation: match[1],
+      key: match[2]
+    });
+  }
+
+  return { fetches, storageKeys };
+}
+
+module.exports = { extractFrontend };

package/src/extractors/functions.js

@@ -0,0 +1,69 @@
+/**
+ * extractFunctions(content, filename) → Array<{ name, params, returnType }>
+ *
+ * Handles both single-line and multiline function signatures.
+ * Only extracts TOP-LEVEL functions (lines starting at column 0).
+ * Skips dunder methods (__name__).
+ * Includes private functions (_name).
+ */
+function extractFunctions(content, filename) {
+  const functions = [];
+  const lines = content.split('\n');
+
+  // Step 1: Collapse multiline signatures
+  const collapsed = [];
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    // Only match top-level function defs (column 0)
+    if (/^(async\s+)?def\s+\w+\s*\(/.test(line)) {
+      let combined = line;
+      // If no closing paren, keep appending (up to 10 lines safety limit)
+      let safety = 0;
+      while (!combined.includes(')') && safety < 10 && i + 1 < lines.length) {
+        i++;
+        safety++;
+        combined += ' ' + lines[i].trim();
+      }
+      collapsed.push(combined);
+    }
+  }
+
+  // Step 2: Extract from collapsed lines
+  const defPattern = /^(async\s+)?def\s+(\w+)\s*\(([^)]*)\)\s*(?:->\s*(.+?))?\s*:/;
+
+  for (const line of collapsed) {
+    const match = line.match(defPattern);
+    if (!match) continue;
+
+    const name = match[2];
+
+    // Skip dunder methods
+    if (name.startsWith('__')) continue;
+
+    const rawParams = match[3];
+    const returnType = match[4] ? match[4].trim() : '\u2014';
+
+    // Step 3: Clean params
+    const skipParams = new Set(['self', '*args', '**kwargs', '*', '']);
+    const params = rawParams
+      .split(',')
+      .map(p => {
+        // Strip type annotation (everything after first ":")
+        let cleaned = p.split(':')[0];
+        // Strip default value (everything after first "=")
+        cleaned = cleaned.split('=')[0];
+        return cleaned.trim();
+      })
+      .filter(p => !skipParams.has(p));
+
+    functions.push({
+      name,
+      params: params.length > 0 ? params.join(', ') : '\u2014',
+      returnType
+    });
+  }
+
+  return functions;
+}
+
+module.exports = { extractFunctions };

package/src/extractors/languages/html.js

@@ -0,0 +1,15 @@
+const { extractFrontend } = require('../frontend');
+
+module.exports = {
+  name: 'html',
+  extensions: ['.html'],
+  extract(content, filename) {
+    try {
+      const { fetches, storageKeys } = extractFrontend(content);
+      return { routes: [], models: [], functions: [], envVars: [], dbTables: [], fetches, storageKeys };
+    } catch (err) {
+      console.warn(`[CARTO] html plugin error on ${filename}: ${err.message}`);
+      return { routes: [], models: [], functions: [], envVars: [], dbTables: [], fetches: [], storageKeys: [] };
+    }
+  }
+};

package/src/extractors/languages/javascript.js

@@ -0,0 +1,376 @@
+const parser = require('@babel/parser');
+const path = require('path');
+
+const PARSE_OPTIONS = {
+  sourceType: 'module',
+  allowImportExportEverywhere: true,
+  allowReturnOutsideFunction: true,
+  plugins: ['jsx', 'optionalChaining', 'nullishCoalescingOperator', 'classProperties', 'decorators-legacy']
+};
+
+// Known Express-like router object names
+const ROUTER_NAMES = new Set(['app', 'router', 'server', 'api']);
+// HTTP methods
+const HTTP_METHODS = new Set(['get', 'post', 'put', 'delete', 'patch']);
+
+module.exports = {
+  name: 'javascript',
+  extensions: ['.js', '.jsx', '.mjs', '.cjs'],
+  extract(content, filename) {
+    let ast;
+    try {
+      ast = parser.parse(content, PARSE_OPTIONS);
+    } catch (err) {
+      console.warn(`[CARTO] JS parse failed on ${filename}: ${err.message} — skipping`);
+      return { routes: [], models: [], functions: [], envVars: [], dbTables: [], fetches: [], storageKeys: [] };
+    }
+
+    return {
+      routes:      extractExpressRoutes(ast, filename),
+      models:      [],
+      functions:   extractJSFunctions(ast),
+      envVars:     extractProcessEnv(ast),
+      dbTables:    [],
+      fetches:     extractJSFetches(ast),
+      storageKeys: [],
+    };
+  },
+
+  // Expose internals for typescript.js to reuse
+  _extractExpressRoutes: extractExpressRoutes,
+  _extractProcessEnv: extractProcessEnv,
+  _extractJSFetches: extractJSFetches,
+  _extractJSFunctions: extractJSFunctions,
+  _PARSE_OPTIONS: PARSE_OPTIONS,
+};
+
+// ---------------------------------------------------------------------------
+// AST traversal helper
+// ---------------------------------------------------------------------------
+
+function walk(node, visitor) {
+  if (!node || typeof node !== 'object') return;
+  if (Array.isArray(node)) {
+    for (const child of node) walk(child, visitor);
+    return;
+  }
+  if (node.type) {
+    visitor(node);
+  }
+  for (const key of Object.keys(node)) {
+    if (key === 'leadingComments' || key === 'trailingComments' || key === 'innerComments') continue;
+    const child = node[key];
+    if (child && typeof child === 'object') {
+      walk(child, visitor);
+    }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Express route extraction
+// ---------------------------------------------------------------------------
+
+function extractExpressRoutes(ast, filename) {
+  const routes = [];
+
+  // Check for Next.js route file pattern
+  const isNextRoute = isNextJSRouteFile(filename);
+
+  if (isNextRoute) {
+    const nextRoutes = extractNextJSRoutes(ast, filename);
+    routes.push(...nextRoutes);
+  }
+
+  walk(ast, (node) => {
+    if (node.type !== 'CallExpression') return;
+    if (!node.callee || node.callee.type !== 'MemberExpression') return;
+
+    const obj = node.callee.object;
+    const prop = node.callee.property;
+
+    // Check: app.get(...), router.post(...), etc.
+    if (!obj || !prop) return;
+    const objName = obj.name || (obj.property && obj.property.name);
+    const methodName = prop.name || (prop.value);
+
+    if (!objName || !ROUTER_NAMES.has(objName)) return;
+    if (!methodName || !HTTP_METHODS.has(methodName)) return;
+
+    // First argument should be the path
+    const args = node.arguments;
+    if (!args || args.length === 0) return;
+
+    let routePath;
+    if (args[0].type === 'StringLiteral') {
+      routePath = args[0].value;
+    } else if (args[0].type === 'TemplateLiteral') {
+      routePath = '[dynamic]';
+    } else {
+      return; // Can't determine path — skip
+    }
+
+    // Handler name: last argument
+    const lastArg = args[args.length - 1];
+    let handler = '[anonymous]';
+    if (lastArg.type === 'Identifier') {
+      handler = lastArg.name;
+    } else if (lastArg.type === 'FunctionExpression' && lastArg.id) {
+      handler = lastArg.id.name;
+    } else if (lastArg.type === 'ArrowFunctionExpression') {
+      // Arrow functions are anonymous, but check if assigned to a variable
+      handler = '[anonymous]';
+    }
+
+    routes.push({
+      method: methodName.toUpperCase(),
+      path: routePath,
+      functionName: handler
+    });
+  });
+
+  return routes;
+}
+
+function isNextJSRouteFile(filename) {
+  const base = path.basename(filename, path.extname(filename));
+  return (base === 'route' || base === 'Route');
+}
+
+function extractNextJSRoutes(ast, filename) {
+  const routes = [];
+  const httpExports = new Set(['GET', 'POST', 'PUT', 'DELETE', 'PATCH']);
+
+  // Infer path from filename: app/api/users/route.js → /api/users
+  // We only have the basename, so we can't infer the full path
+  // This would need the full file path — for now use the filename
+  const routePath = '[inferred]';
+
+  walk(ast, (node) => {
+    // export async function GET(req) { ... }
+    if (node.type === 'ExportNamedDeclaration' && node.declaration) {
+      const decl = node.declaration;
+      if (decl.type === 'FunctionDeclaration' && decl.id && httpExports.has(decl.id.name)) {
+        routes.push({
+          method: decl.id.name,
+          path: routePath,
+          functionName: decl.id.name
+        });
+      }
+      // export const GET = async (req) => { ... }
+      if (decl.type === 'VariableDeclaration') {
+        for (const declarator of decl.declarations) {
+          if (declarator.id && declarator.id.name && httpExports.has(declarator.id.name)) {
+            routes.push({
+              method: declarator.id.name,
+              path: routePath,
+              functionName: declarator.id.name
+            });
+          }
+        }
+      }
+    }
+  });
+
+  return routes;
+}
+
+// ---------------------------------------------------------------------------
+// JS function extraction
+// ---------------------------------------------------------------------------
+
+function extractJSFunctions(ast) {
+  const functions = [];
+
+  if (!ast.program || !ast.program.body) return functions;
+
+  for (const node of ast.program.body) {
+    // 1. FunctionDeclaration at top level
+    if (node.type === 'FunctionDeclaration' && node.id) {
+      const name = node.id.name;
+      if (shouldSkipJSFunction(name, node.params)) continue;
+      functions.push({
+        name,
+        params: extractParams(node.params),
+        returnType: '\u2014'
+      });
+    }
+
+    // 2. VariableDeclaration at top level with arrow/function expression
+    if (node.type === 'VariableDeclaration') {
+      for (const decl of node.declarations) {
+        if (!decl.id || !decl.id.name) continue;
+        if (!decl.init) continue;
+        if (decl.init.type === 'ArrowFunctionExpression' || decl.init.type === 'FunctionExpression') {
+          const name = decl.id.name;
+          if (shouldSkipJSFunction(name, decl.init.params)) continue;
+          functions.push({
+            name,
+            params: extractParams(decl.init.params),
+            returnType: '\u2014'
+          });
+        }
+      }
+    }
+
+    // 3. ExportDefaultDeclaration wrapping a function
+    if (node.type === 'ExportDefaultDeclaration' && node.declaration) {
+      const decl = node.declaration;
+      if (decl.type === 'FunctionDeclaration' && decl.id) {
+        const name = decl.id.name;
+        if (!shouldSkipJSFunction(name, decl.params)) {
+          functions.push({
+            name,
+            params: extractParams(decl.params),
+            returnType: '\u2014'
+          });
+        }
+      }
+    }
+
+    // 4. ExportNamedDeclaration wrapping a function or variable
+    if (node.type === 'ExportNamedDeclaration' && node.declaration) {
+      const decl = node.declaration;
+      if (decl.type === 'FunctionDeclaration' && decl.id) {
+        const name = decl.id.name;
+        if (!shouldSkipJSFunction(name, decl.params)) {
+          functions.push({
+            name,
+            params: extractParams(decl.params),
+            returnType: '\u2014'
+          });
+        }
+      }
+      if (decl.type === 'VariableDeclaration') {
+        for (const vDecl of decl.declarations) {
+          if (!vDecl.id || !vDecl.id.name || !vDecl.init) continue;
+          if (vDecl.init.type === 'ArrowFunctionExpression' || vDecl.init.type === 'FunctionExpression') {
+            const name = vDecl.id.name;
+            if (!shouldSkipJSFunction(name, vDecl.init.params)) {
+              functions.push({
+                name,
+                params: extractParams(vDecl.init.params),
+                returnType: '\u2014'
+              });
+            }
+          }
+        }
+      }
+    }
+  }
+
+  return functions;
+}
+
+function shouldSkipJSFunction(name, params) {
+  // Skip _ prefixed functions only if they have 0 params (likely private utility)
+  if (name.startsWith('_') && (!params || params.length === 0)) return true;
+  return false;
+}
+
+function extractParams(params) {
+  if (!params || params.length === 0) return '\u2014';
+  const names = [];
+  for (const p of params) {
+    if (p.type === 'Identifier') {
+      names.push(p.name);
+    } else if (p.type === 'AssignmentPattern' && p.left && p.left.type === 'Identifier') {
+      names.push(p.left.name);
+    } else if (p.type === 'RestElement' && p.argument && p.argument.type === 'Identifier') {
+      // ...args — skip like Python's *args
+      continue;
+    } else if (p.type === 'ObjectPattern') {
+      names.push('{...}');
+    } else if (p.type === 'ArrayPattern') {
+      names.push('[...]');
+    }
+  }
+  return names.length > 0 ? names.join(', ') : '\u2014';
+}
+
+// ---------------------------------------------------------------------------
+// process.env extraction
+// ---------------------------------------------------------------------------
+
+function extractProcessEnv(ast) {
+  const vars = new Set();
+
+  walk(ast, (node) => {
+    if (node.type !== 'MemberExpression') return;
+
+    // Pattern 1: process.env.VAR_NAME
+    if (
+      node.object &&
+      node.object.type === 'MemberExpression' &&
+      node.object.object &&
+      node.object.object.name === 'process' &&
+      node.object.property &&
+      node.object.property.name === 'env' &&
+      node.property
+    ) {
+      if (!node.computed && node.property.name) {
+        vars.add(node.property.name);
+      }
+      // Pattern 2: process.env['VAR_NAME']
+      if (node.computed && node.property.type === 'StringLiteral') {
+        vars.add(node.property.value);
+      }
+    }
+  });
+
+  return [...vars].sort();
+}
+
+// ---------------------------------------------------------------------------
+// fetch() extraction
+// ---------------------------------------------------------------------------
+
+function extractJSFetches(ast) {
+  const fetches = [];
+
+  walk(ast, (node) => {
+    if (node.type !== 'CallExpression') return;
+
+    // fetch(...) or something.fetch(...)
+    const isFetch =
+      (node.callee.type === 'Identifier' && node.callee.name === 'fetch') ||
+      (node.callee.type === 'MemberExpression' && node.callee.property && node.callee.property.name === 'fetch');
+
+    if (!isFetch) return;
+    if (!node.arguments || node.arguments.length === 0) return;
+
+    const firstArg = node.arguments[0];
+    let url = '[dynamic]';
+    let method = '[dynamic]';
+
+    if (firstArg.type === 'StringLiteral') {
+      url = firstArg.value;
+      method = 'GET'; // default
+    } else if (firstArg.type === 'TemplateLiteral') {
+      url = '[dynamic]';
+      method = '[dynamic]';
+    } else if (firstArg.type === 'Identifier') {
+      url = '[dynamic]';
+      method = '[dynamic]';
+    } else {
+      return;
+    }
+
+    // Check for method in second argument (options object)
+    if (node.arguments.length > 1 && node.arguments[1].type === 'ObjectExpression') {
+      for (const prop of node.arguments[1].properties) {
+        if (
+          prop.key &&
+          (prop.key.name === 'method' || prop.key.value === 'method') &&
+          prop.value &&
+          prop.value.type === 'StringLiteral'
+        ) {
+          method = prop.value.value.toUpperCase();
+        }
+      }
+    }
+
+    fetches.push({ url, method });
+  });
+
+  return fetches;
+}