carto-md 1.0.0 → 1.0.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "carto-md",
-  "version": "1.0.0",
+  "version": "1.0.2",
   "description": "The context layer for AI-native development.",
   "bin": {
     "carto": "src/cli/index.js"

package/src/agents/validator.js

@@ -0,0 +1,88 @@
+/**
+ * validateExtracted(data) → cleaned data
+ *
+ * Runs after extraction, before formatting.
+ * Drops anything that looks wrong. Never throws.
+ */
+function validateExtracted({ routes, models, functions, envVars, dbTables }) {
+  return {
+    routes:    validateRoutes(routes || []),
+    models:    validateModels(models || []),
+    functions: validateFunctions(functions || {}),
+    envVars:   validateEnvVars(envVars || []),
+    dbTables:  validateDBTables(dbTables || [])
+  };
+}
+
+const VALID_METHODS = new Set(['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'HEAD', 'OPTIONS']);
+const VALID_HANDLER = /^[a-zA-Z_]\w*$|^\[anonymous\]$/;
+const VALID_CLASS_NAME = /^[A-Z][a-zA-Z0-9]*$/;
+const VALID_FIELD_NAME = /^[a-z_]\w*$/;
+const VALID_FUNC_NAME = /^_?[a-zA-Z][a-zA-Z0-9_]*$/;
+const VALID_ENV_VAR = /^[A-Z][A-Z0-9_]*$/;
+const VALID_TABLE_NAME = /^[a-z][a-z0-9_]*$/;
+
+function validateRoutes(routes) {
+  return routes.filter(r => {
+    if (!r.method || !VALID_METHODS.has(r.method)) return false;
+    if (!r.path || (!r.path.startsWith('/') && r.path !== '[dynamic]' && r.path !== '[inferred]')) return false;
+    if (!r.functionName || !VALID_HANDLER.test(r.functionName)) return false;
+    return true;
+  });
+}
+
+function validateModels(models) {
+  return models
+    .map(m => {
+      if (!m.className || !VALID_CLASS_NAME.test(m.className)) return null;
+      if (!Array.isArray(m.fields)) return null;
+
+      const validFields = m.fields.filter(f => {
+        if (!f.name || !VALID_FIELD_NAME.test(f.name)) return false;
+        return true;
+      });
+
+      if (validFields.length === 0) return null;
+      return { className: m.className, fields: validFields };
+    })
+    .filter(Boolean);
+}
+
+function validateFunctions(functionsMap) {
+  const cleaned = {};
+
+  for (const [filename, funcs] of Object.entries(functionsMap)) {
+    const validFuncs = funcs.filter(f => {
+      // Name must be a valid identifier
+      if (!f.name || !VALID_FUNC_NAME.test(f.name)) return false;
+      // Name must not contain spaces, brackets, commas
+      if (/[\s\[\],]/.test(f.name)) return false;
+      // Params must not contain parse artifacts
+      if (f.params && (/\[\[/.test(f.params) || /\]\]/.test(f.params) || /Any\]\]/.test(f.params))) return false;
+      return true;
+    });
+
+    if (validFuncs.length > 0) {
+      cleaned[filename] = validFuncs;
+    }
+  }
+
+  return cleaned;
+}
+
+function validateEnvVars(envVars) {
+  return envVars.filter(v => {
+    if (!v.name || !VALID_ENV_VAR.test(v.name)) return false;
+    return true;
+  });
+}
+
+function validateDBTables(dbTables) {
+  return dbTables.filter(t => {
+    if (!t.tableName || !VALID_TABLE_NAME.test(t.tableName)) return false;
+    if (!t.modelName || !VALID_CLASS_NAME.test(t.modelName)) return false;
+    return true;
+  });
+}
+
+module.exports = { validateExtracted };

package/src/extractors/frontend.js

@@ -37,7 +37,26 @@ function extractFrontend(content) {
     });
   }
 
-  return { fetches, storageKeys };
+  // Collapse dynamic fetches into a single summary row
+  const staticFetches = fetches.filter(f => f.url !== '[dynamic]');
+  const dynamicCount = fetches.filter(f => f.url === '[dynamic]').length;
+  if (dynamicCount > 0) {
+    staticFetches.push({
+      url: `dynamic calls detected (${dynamicCount} unresolved)`,
+      method: '\u2014'
+    });
+  }
+
+  // Deduplicate storage keys — keep first occurrence only
+  const seen = new Set();
+  const dedupedStorageKeys = storageKeys.filter(({ operation, key }) => {
+    const id = `${operation}::${key}`;
+    if (seen.has(id)) return false;
+    seen.add(id);
+    return true;
+  });
+
+  return { fetches: staticFetches, storageKeys: dedupedStorageKeys };
 }
 
 module.exports = { extractFrontend };

package/src/extractors/functions.js

@@ -43,15 +43,16 @@ function extractFunctions(content, filename) {
     const rawParams = match[3];
     const returnType = match[4] ? match[4].trim() : '\u2014';
 
-    // Step 3: Clean params
-    const skipParams = new Set(['self', '*args', '**kwargs', '*', '']);
-    const params = rawParams
-      .split(',')
+    // Step 3: Clean params — use smart split to handle commas inside brackets
+    const skipParams = new Set(['self', '']);
+    const params = splitParams(rawParams)
       .map(p => {
         // Strip type annotation (everything after first ":")
         let cleaned = p.split(':')[0];
         // Strip default value (everything after first "=")
         cleaned = cleaned.split('=')[0];
+        // Strip leading * or **
+        cleaned = cleaned.replace(/^\*{1,2}/, '');
         return cleaned.trim();
       })
       .filter(p => !skipParams.has(p));
@@ -66,4 +67,26 @@ function extractFunctions(content, filename) {
   return functions;
 }
 
+/**
+ * Splits params by comma at the top level only.
+ * Commas inside brackets [], parentheses (), or braces {} do NOT split.
+ */
+function splitParams(rawParams) {
+  const params = [];
+  let depth = 0;
+  let current = '';
+  for (const char of rawParams) {
+    if (char === '[' || char === '(' || char === '{') depth++;
+    else if (char === ']' || char === ')' || char === '}') depth--;
+    else if (char === ',' && depth === 0) {
+      params.push(current.trim());
+      current = '';
+      continue;
+    }
+    current += char;
+  }
+  if (current.trim()) params.push(current.trim());
+  return params;
+}
+
 module.exports = { extractFunctions };

package/src/sync.js

@@ -4,8 +4,9 @@ const { loadLanguagePlugins, getPluginForFile } = require('./extractors/loader')
 const { formatSections } = require('./agents/formatter');
 const { mergeIntoAgentsMd } = require('./agents/merger');
 const { inferResponsibility } = require('./extractors/filemap');
+const { validateExtracted } = require('./agents/validator');
 
-const IGNORE_DIRS = new Set(['node_modules', '.git', '__pycache__', '.venv', 'venv', '.idea', '.vscode', '.carto']);
+const IGNORE_DIRS = new Set(['node_modules', '.git', '__pycache__', '.venv', 'venv', '.idea', '.vscode', '.carto', 'AGENTS.md']);
 
 // Load plugins once at module load
 const plugins = loadLanguagePlugins();
@@ -142,6 +143,29 @@ async function runFullSync(config) {
     allStorageKeys = allStorageKeys.concat(result.storageKeys);
   }
 
+  // Global dedup: collapse dynamic fetches across all files into one summary row
+  const staticFetches = allFetches.filter(f => f.url !== '[dynamic]' && !f.url.startsWith('dynamic calls detected'));
+  let totalDynamic = 0;
+  for (const f of allFetches) {
+    if (f.url === '[dynamic]') totalDynamic++;
+    // Also count already-collapsed per-file rows
+    const m = f.url.match(/^dynamic calls detected \((\d+) unresolved\)$/);
+    if (m) totalDynamic += parseInt(m[1], 10);
+  }
+  if (totalDynamic > 0) {
+    staticFetches.push({ url: `dynamic calls detected (${totalDynamic} unresolved)`, method: '\u2014' });
+  }
+  allFetches = staticFetches;
+
+  // Global dedup: storage keys across all files
+  const skSeen = new Set();
+  allStorageKeys = allStorageKeys.filter(({ operation, key }) => {
+    const id = `${operation}::${key}`;
+    if (skSeen.has(id)) return false;
+    skSeen.add(id);
+    return true;
+  });
+
   // Build file map
   const fileMap = [];
   for (const filePath of allCodeFiles) {
@@ -162,16 +186,25 @@ async function runFullSync(config) {
   // Scan project structure
   const structure = await scanStructure(config.projectRoot);
 
-  const autoContent = formatSections({
+  // Validate extracted data — drop anything malformed
+  const validated = validateExtracted({
     routes: allRoutes,
     models: allModels,
+    functions: functionsMap,
+    envVars,
+    dbTables: dbTableList
+  });
+
+  const autoContent = formatSections({
+    routes: validated.routes,
+    models: validated.models,
     frontend: { fetches: allFetches, storageKeys: allStorageKeys },
     structure,
     warnings,
     fileMap,
-    functions: functionsMap,
-    dbTables: dbTableList,
-    envVars
+    functions: validated.functions,
+    dbTables: validated.dbTables,
+    envVars: validated.envVars
   });
 
   mergeIntoAgentsMd(config.output, autoContent);