carto-md 1.0.1 → 1.0.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "carto-md",
-  "version": "1.0.1",
+  "version": "1.0.3",
   "description": "The context layer for AI-native development.",
   "bin": {
     "carto": "src/cli/index.js"

package/src/agents/formatter.js

@@ -9,10 +9,11 @@
  *   5. Functions (auto)
  *   6. Database Tables (auto)
  *   7. Environment Variables (auto)
- *   8. Frontend API Calls (auto)
- *   9. Frontend Storage Keys (auto)
+ *   8. File Relationships (auto)
+ *   9. Frontend API Calls (auto)
+ *   10. Frontend Storage Keys (auto)
  */
-function formatSections({ routes, models, frontend, structure, warnings, fileMap, functions, dbTables, envVars }) {
+function formatSections({ routes, models, frontend, structure, warnings, fileMap, functions, dbTables, envVars, importGraph }) {
   const sections = [];
 
   // 1. Project Structure
@@ -114,7 +115,21 @@ function formatSections({ routes, models, frontend, structure, warnings, fileMap
     sections.push('_No environment variables detected._');
   }
 
-  // 8. Frontend API Calls
+  // 8. File Relationships
+  sections.push('\n## File Relationships (auto)\n');
+  if (importGraph && Object.keys(importGraph).length > 0) {
+    const sortedFiles = Object.keys(importGraph).sort();
+    for (const file of sortedFiles) {
+      const deps = importGraph[file];
+      if (deps.length > 0) {
+        sections.push(`${file} \u2192 ${deps.join(', ')}`);
+      }
+    }
+  } else {
+    sections.push('_No file relationships detected._');
+  }
+
+  // 9. Frontend API Calls
   sections.push('\n## Frontend API Calls (auto)\n');
   if (frontend.fetches.length > 0) {
     sections.push('| Method | URL |');
@@ -126,7 +141,7 @@ function formatSections({ routes, models, frontend, structure, warnings, fileMap
     sections.push('_No fetch calls found._');
   }
 
-  // 9. Frontend Storage Keys
+  // 10. Frontend Storage Keys
   sections.push('\n## Frontend Storage Keys (auto)\n');
   if (frontend.storageKeys.length > 0) {
     sections.push('| Operation | Key |');

package/src/agents/validator.js

@@ -0,0 +1,88 @@
+/**
+ * validateExtracted(data) → cleaned data
+ *
+ * Runs after extraction, before formatting.
+ * Drops anything that looks wrong. Never throws.
+ */
+function validateExtracted({ routes, models, functions, envVars, dbTables }) {
+  return {
+    routes:    validateRoutes(routes || []),
+    models:    validateModels(models || []),
+    functions: validateFunctions(functions || {}),
+    envVars:   validateEnvVars(envVars || []),
+    dbTables:  validateDBTables(dbTables || [])
+  };
+}
+
+const VALID_METHODS = new Set(['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'HEAD', 'OPTIONS']);
+const VALID_HANDLER = /^[a-zA-Z_]\w*$|^\[anonymous\]$/;
+const VALID_CLASS_NAME = /^[A-Z][a-zA-Z0-9]*$/;
+const VALID_FIELD_NAME = /^[a-z_]\w*$/;
+const VALID_FUNC_NAME = /^_?[a-zA-Z][a-zA-Z0-9_]*$/;
+const VALID_ENV_VAR = /^[A-Z][A-Z0-9_]*$/;
+const VALID_TABLE_NAME = /^[a-z][a-z0-9_]*$/;
+
+function validateRoutes(routes) {
+  return routes.filter(r => {
+    if (!r.method || !VALID_METHODS.has(r.method)) return false;
+    if (!r.path || (!r.path.startsWith('/') && r.path !== '[dynamic]' && r.path !== '[inferred]')) return false;
+    if (!r.functionName || !VALID_HANDLER.test(r.functionName)) return false;
+    return true;
+  });
+}
+
+function validateModels(models) {
+  return models
+    .map(m => {
+      if (!m.className || !VALID_CLASS_NAME.test(m.className)) return null;
+      if (!Array.isArray(m.fields)) return null;
+
+      const validFields = m.fields.filter(f => {
+        if (!f.name || !VALID_FIELD_NAME.test(f.name)) return false;
+        return true;
+      });
+
+      if (validFields.length === 0) return null;
+      return { className: m.className, fields: validFields };
+    })
+    .filter(Boolean);
+}
+
+function validateFunctions(functionsMap) {
+  const cleaned = {};
+
+  for (const [filename, funcs] of Object.entries(functionsMap)) {
+    const validFuncs = funcs.filter(f => {
+      // Name must be a valid identifier
+      if (!f.name || !VALID_FUNC_NAME.test(f.name)) return false;
+      // Name must not contain spaces, brackets, commas
+      if (/[\s\[\],]/.test(f.name)) return false;
+      // Params must not contain parse artifacts
+      if (f.params && (/\[\[/.test(f.params) || /\]\]/.test(f.params) || /Any\]\]/.test(f.params))) return false;
+      return true;
+    });
+
+    if (validFuncs.length > 0) {
+      cleaned[filename] = validFuncs;
+    }
+  }
+
+  return cleaned;
+}
+
+function validateEnvVars(envVars) {
+  return envVars.filter(v => {
+    if (!v.name || !VALID_ENV_VAR.test(v.name)) return false;
+    return true;
+  });
+}
+
+function validateDBTables(dbTables) {
+  return dbTables.filter(t => {
+    if (!t.tableName || !VALID_TABLE_NAME.test(t.tableName)) return false;
+    if (!t.modelName || !VALID_CLASS_NAME.test(t.modelName)) return false;
+    return true;
+  });
+}
+
+module.exports = { validateExtracted };

package/src/extractors/frontend.js

@@ -42,7 +42,7 @@ function extractFrontend(content) {
   const dynamicCount = fetches.filter(f => f.url === '[dynamic]').length;
   if (dynamicCount > 0) {
     staticFetches.push({
-      url: `[dynamic \u00d7${dynamicCount}]`,
+      url: `dynamic calls detected (${dynamicCount} unresolved)`,
       method: '\u2014'
     });
   }

package/src/extractors/imports.js

@@ -0,0 +1,212 @@
+const path = require('path');
+const fs = require('fs');
+
+/**
+ * extractImports(content, filePath, projectRoot) → Array<string>
+ *
+ * Extracts relative import paths from a source file.
+ * Returns resolved relative paths (from project root) of local dependencies.
+ *
+ * JS/TS patterns:
+ *   import X from './Y'
+ *   import { X } from './Y'
+ *   import './Y'
+ *   const X = require('./Y')
+ *   require('./Y')
+ *
+ * Python patterns:
+ *   from .module import X        (relative)
+ *   from ..module import X       (relative)
+ *   from app.module import X     (local package — resolved if file exists)
+ *   import .module               (relative)
+ *
+ * Only includes paths that resolve to actual files in the project.
+ * Skips: node_modules, non-code files, anything that doesn't resolve.
+ */
+function extractImports(content, filePath, projectRoot) {
+  const ext = path.extname(filePath).toLowerCase();
+  const fileDir = path.dirname(filePath);
+
+  let rawImports = [];
+
+  if (['.js', '.jsx', '.ts', '.tsx', '.mjs', '.cjs'].includes(ext)) {
+    rawImports = extractJSImports(content);
+  } else if (ext === '.py') {
+    rawImports = extractPythonImports(content, filePath, projectRoot);
+  }
+
+  // Resolve and deduplicate
+  const resolved = new Set();
+
+  for (const imp of rawImports) {
+    const resolvedPath = resolveImportPath(imp, fileDir, projectRoot, ext);
+    if (resolvedPath) {
+      // Store as relative to project root
+      const rel = path.relative(projectRoot, resolvedPath);
+      resolved.add(rel);
+    }
+  }
+
+  return [...resolved].sort();
+}
+
+/**
+ * Extract import paths from JS/TS content. Relative paths only.
+ */
+function extractJSImports(content) {
+  const imports = [];
+
+  // import ... from './path'  or  import './path'
+  const importPattern = /import\s+(?:[\s\S]*?\s+from\s+)?['"](\.[^'"]+)['"]/g;
+  let match;
+  while ((match = importPattern.exec(content)) !== null) {
+    imports.push(match[1]);
+  }
+
+  // require('./path')
+  const requirePattern = /require\s*\(\s*['"](\.[^'"]+)['"]\s*\)/g;
+  while ((match = requirePattern.exec(content)) !== null) {
+    imports.push(match[1]);
+  }
+
+  return imports;
+}
+
+/**
+ * Extract import paths from Python content. Relative imports only.
+ */
+function extractPythonImports(content, filePath, projectRoot) {
+  const imports = [];
+  const fileDir = path.dirname(filePath);
+
+  // from .module import X  or  from ..module import X
+  const fromRelPattern = /^from\s+(\.+\w*(?:\.\w+)*)\s+import/gm;
+  let match;
+  while ((match = fromRelPattern.exec(content)) !== null) {
+    const resolved = resolvePythonRelativeImport(match[1], fileDir);
+    if (resolved) imports.push(resolved);
+  }
+
+  // from app.module import X — try to resolve as local file
+  const fromAbsPattern = /^from\s+(\w+(?:\.\w+)+)\s+import/gm;
+  while ((match = fromAbsPattern.exec(content)) !== null) {
+    const modulePath = match[1].replace(/\./g, path.sep);
+    // Try from project root
+    const resolved = tryResolvePythonModule(modulePath, projectRoot);
+    if (resolved) {
+      imports.push(resolved);
+      continue;
+    }
+    // Try from file's directory (for cases like `from app.models` when inside aws-risk-agent/)
+    const fromFileDir = tryResolvePythonModule(modulePath, fileDir);
+    if (fromFileDir) {
+      imports.push(fromFileDir);
+      continue;
+    }
+    // Try from parent directories up to project root
+    let searchDir = path.dirname(fileDir);
+    while (searchDir.startsWith(projectRoot) && searchDir !== projectRoot) {
+      const fromParent = tryResolvePythonModule(modulePath, searchDir);
+      if (fromParent) {
+        imports.push(fromParent);
+        break;
+      }
+      searchDir = path.dirname(searchDir);
+    }
+  }
+
+  return imports;
+}
+
+/**
+ * Try to resolve a dotted Python module path from a base directory.
+ */
+function tryResolvePythonModule(modulePath, baseDir) {
+  const asFile = path.join(baseDir, modulePath + '.py');
+  if (fs.existsSync(asFile)) return asFile;
+  const asInit = path.join(baseDir, modulePath, '__init__.py');
+  if (fs.existsSync(asInit)) return asInit;
+  return null;
+}
+
+/**
+ * Resolve a Python relative import like '.models' or '..utils' to an absolute path.
+ */
+function resolvePythonRelativeImport(importStr, fileDir) {
+  // Count leading dots
+  let dots = 0;
+  while (dots < importStr.length && importStr[dots] === '.') dots++;
+
+  const modulePart = importStr.substring(dots);
+
+  // Go up (dots - 1) directories from fileDir
+  let baseDir = fileDir;
+  for (let i = 1; i < dots; i++) {
+    baseDir = path.dirname(baseDir);
+  }
+
+  if (!modulePart) return null;
+
+  const modulePath = modulePart.replace(/\./g, path.sep);
+  const asFile = path.join(baseDir, modulePath + '.py');
+  if (fs.existsSync(asFile)) return asFile;
+
+  const asInit = path.join(baseDir, modulePath, '__init__.py');
+  if (fs.existsSync(asInit)) return asInit;
+
+  return null;
+}
+
+/**
+ * Resolve a JS/TS import path to an actual file.
+ * Tries: exact, .js, .ts, .jsx, .tsx, /index.js, /index.ts
+ */
+function resolveImportPath(importPath, fileDir, projectRoot, sourceExt) {
+  // For Python, importPath is already absolute
+  if (path.isAbsolute(importPath)) {
+    return fs.existsSync(importPath) ? importPath : null;
+  }
+
+  const base = path.resolve(fileDir, importPath);
+
+  // Try exact
+  if (fs.existsSync(base) && fs.statSync(base).isFile()) return base;
+
+  // Try extensions
+  const extensions = ['.js', '.ts', '.jsx', '.tsx', '.mjs', '.cjs'];
+  for (const ext of extensions) {
+    const withExt = base + ext;
+    if (fs.existsSync(withExt)) return withExt;
+  }
+
+  // Try index files
+  for (const ext of extensions) {
+    const indexFile = path.join(base, 'index' + ext);
+    if (fs.existsSync(indexFile)) return indexFile;
+  }
+
+  return null;
+}
+
+/**
+ * buildImportGraph(fileContents, projectRoot) → { 'relative/path.js': ['relative/dep.js', ...] }
+ *
+ * fileContents: Array of { filePath, content } (absolute paths)
+ * Returns a map of relative file paths to their relative dependencies.
+ * Only includes files that have at least one resolved dependency.
+ */
+function buildImportGraph(fileContents, projectRoot) {
+  const graph = {};
+
+  for (const { filePath, content } of fileContents) {
+    const deps = extractImports(content, filePath, projectRoot);
+    if (deps.length > 0) {
+      const relPath = path.relative(projectRoot, filePath);
+      graph[relPath] = deps;
+    }
+  }
+
+  return graph;
+}
+
+module.exports = { extractImports, buildImportGraph };

package/src/sync.js

@@ -4,8 +4,10 @@ const { loadLanguagePlugins, getPluginForFile } = require('./extractors/loader')
 const { formatSections } = require('./agents/formatter');
 const { mergeIntoAgentsMd } = require('./agents/merger');
 const { inferResponsibility } = require('./extractors/filemap');
+const { validateExtracted } = require('./agents/validator');
+const { buildImportGraph } = require('./extractors/imports');
 
-const IGNORE_DIRS = new Set(['node_modules', '.git', '__pycache__', '.venv', 'venv', '.idea', '.vscode', '.carto']);
+const IGNORE_DIRS = new Set(['node_modules', '.git', '__pycache__', '.venv', 'venv', '.idea', '.vscode', '.carto', 'AGENTS.md']);
 
 // Load plugins once at module load
 const plugins = loadLanguagePlugins();
@@ -143,16 +145,16 @@ async function runFullSync(config) {
   }
 
   // Global dedup: collapse dynamic fetches across all files into one summary row
-  const staticFetches = allFetches.filter(f => !f.url.startsWith('[dynamic'));
+  const staticFetches = allFetches.filter(f => f.url !== '[dynamic]' && !f.url.startsWith('dynamic calls detected'));
   let totalDynamic = 0;
   for (const f of allFetches) {
     if (f.url === '[dynamic]') totalDynamic++;
-    // Also count already-collapsed per-file rows like "[dynamic ×1]"
-    const m = f.url.match(/^\[dynamic\s*\u00d7(\d+)\]$/);
+    // Also count already-collapsed per-file rows
+    const m = f.url.match(/^dynamic calls detected \((\d+) unresolved\)$/);
     if (m) totalDynamic += parseInt(m[1], 10);
   }
   if (totalDynamic > 0) {
-    staticFetches.push({ url: `[dynamic \u00d7${totalDynamic}]`, method: '\u2014' });
+    staticFetches.push({ url: `dynamic calls detected (${totalDynamic} unresolved)`, method: '\u2014' });
   }
   allFetches = staticFetches;
 
@@ -165,6 +167,20 @@ async function runFullSync(config) {
     return true;
   });
 
+  // Build import graph from all processed files
+  const fileContentsForImports = [];
+  const allProcessedPaths = [...new Set([...allCodeFiles, ...allFrontendFiles])];
+  // Re-read is avoided — collect during processing. Use a second pass for simplicity.
+  for (const filePath of allProcessedPaths) {
+    try {
+      const content = fs.readFileSync(filePath, 'utf-8');
+      fileContentsForImports.push({ filePath, content });
+    } catch {
+      // skip — already warned during extraction
+    }
+  }
+  const importGraph = buildImportGraph(fileContentsForImports, config.projectRoot);
+
   // Build file map
   const fileMap = [];
   for (const filePath of allCodeFiles) {
@@ -185,16 +201,26 @@ async function runFullSync(config) {
   // Scan project structure
   const structure = await scanStructure(config.projectRoot);
 
-  const autoContent = formatSections({
+  // Validate extracted data — drop anything malformed
+  const validated = validateExtracted({
     routes: allRoutes,
     models: allModels,
+    functions: functionsMap,
+    envVars,
+    dbTables: dbTableList
+  });
+
+  const autoContent = formatSections({
+    routes: validated.routes,
+    models: validated.models,
     frontend: { fetches: allFetches, storageKeys: allStorageKeys },
     structure,
     warnings,
     fileMap,
-    functions: functionsMap,
-    dbTables: dbTableList,
-    envVars
+    functions: validated.functions,
+    dbTables: validated.dbTables,
+    envVars: validated.envVars,
+    importGraph
   });
 
   mergeIntoAgentsMd(config.output, autoContent);