carto-md 1.0.0

package/.cartoignore

@@ -0,0 +1,16 @@
+# Carto ignore rules — files matching these patterns are excluded from scanning
+# Default rules are always active even without this file
+
+# Secrets and credentials
+.env
+.env.*
+*secret*
+*SECRET*
+*password*
+*PASSWORD*
+*credential*
+*CREDENTIAL*
+*private_key*
+*PRIVATE_KEY*
+*.pem
+*.key

package/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "carto-md",
+  "version": "1.0.0",
+  "description": "The context layer for AI-native development.",
+  "bin": {
+    "carto": "src/cli/index.js"
+  },
+  "main": "./src/sync.js",
+  "scripts": {
+    "test": "node test/test.js"
+  },
+  "dependencies": {
+    "@babel/parser": "^7.29.3",
+    "chokidar": "^3.6.0"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "author": {
+    "name": "Ansh Sonkar",
+    "url": "https://github.com/theanshsonkar",
+    "twitter": "https://x.com/theanshsonkar"
+  },
+  "license": "MIT",
+  "keywords": [
+    "agents",
+    "AGENTS.md",
+    "AI",
+    "context",
+    "codebase",
+    "developer-tools",
+    "cursor",
+    "copilot"
+  ]
+}

package/src/agents/formatter.js

@@ -0,0 +1,150 @@
+/**
+ * Converts extracted data into markdown sections for AGENTS.md.
+ *
+ * Section order:
+ *   1. Project Structure (auto)
+ *   2. File Map (auto)
+ *   3. API Routes (auto)
+ *   4. Models (auto)
+ *   5. Functions (auto)
+ *   6. Database Tables (auto)
+ *   7. Environment Variables (auto)
+ *   8. Frontend API Calls (auto)
+ *   9. Frontend Storage Keys (auto)
+ */
+function formatSections({ routes, models, frontend, structure, warnings, fileMap, functions, dbTables, envVars }) {
+  const sections = [];
+
+  // 1. Project Structure
+  sections.push('## Project Structure (auto)\n');
+  if (structure.length > 0) {
+    for (const entry of structure) {
+      const icon = entry.type === 'dir' ? '📁' : '📄';
+      const suffix = entry.type === 'dir' ? '/' : '';
+      sections.push(`- ${icon} ${entry.name}${suffix}`);
+    }
+  } else {
+    sections.push('_No structure data available._');
+  }
+
+  // 2. File Map
+  sections.push('\n## File Map (auto)\n');
+  if (fileMap && fileMap.length > 0) {
+    sections.push('| File | Responsibility |');
+    sections.push('|------|----------------|');
+    for (const entry of fileMap) {
+      sections.push(`| ${entry.file} | ${entry.responsibility} |`);
+    }
+  } else {
+    sections.push('_No file map data available._');
+  }
+
+  // 3. API Routes
+  sections.push('\n## API Routes (auto)\n');
+  if (routes.length > 0) {
+    sections.push('| Method | Path | Handler |');
+    sections.push('|--------|------|---------|');
+    for (const r of routes) {
+      sections.push(`| ${r.method} | ${r.path} | ${r.functionName} |`);
+    }
+  } else {
+    sections.push('_No routes found._');
+  }
+
+  // 4. Models
+  sections.push('\n## Models (auto)\n');
+  if (models.length > 0) {
+    for (const m of models) {
+      sections.push(`### ${m.className}`);
+      if (m.fields.length > 0) {
+        sections.push('| Field | Type |');
+        sections.push('|-------|------|');
+        for (const f of m.fields) {
+          sections.push(`| ${f.name} | ${f.type} |`);
+        }
+      } else {
+        sections.push('_No fields._');
+      }
+      sections.push('');
+    }
+  } else {
+    sections.push('_No models found._');
+  }
+
+  // 5. Functions
+  sections.push('## Functions (auto)\n');
+  if (functions && Object.keys(functions).length > 0) {
+    const sortedFiles = Object.keys(functions).sort();
+    for (const filename of sortedFiles) {
+      const funcs = functions[filename];
+      if (funcs.length === 0) continue;
+      sections.push(`### ${filename}`);
+      sections.push('| Function | Params | Returns |');
+      sections.push('|----------|--------|---------|');
+      for (const f of funcs) {
+        sections.push(`| ${f.name} | ${f.params} | ${f.returnType} |`);
+      }
+      sections.push('');
+    }
+  } else {
+    sections.push('_No functions found._');
+  }
+
+  // 6. Database Tables
+  sections.push('## Database Tables (auto)\n');
+  if (dbTables && dbTables.length > 0) {
+    sections.push('| Table | Model | File |');
+    sections.push('|-------|-------|------|');
+    for (const t of dbTables) {
+      sections.push(`| ${t.tableName} | ${t.modelName} | ${t.file} |`);
+    }
+  } else {
+    sections.push('_No database tables detected._');
+  }
+
+  // 7. Environment Variables
+  sections.push('\n## Environment Variables (auto)\n');
+  if (envVars && envVars.length > 0) {
+    sections.push('| Variable | Used In |');
+    sections.push('|----------|---------|');
+    for (const v of envVars) {
+      sections.push(`| ${v.name} | ${v.files.join(', ')} |`);
+    }
+  } else {
+    sections.push('_No environment variables detected._');
+  }
+
+  // 8. Frontend API Calls
+  sections.push('\n## Frontend API Calls (auto)\n');
+  if (frontend.fetches.length > 0) {
+    sections.push('| Method | URL |');
+    sections.push('|--------|-----|');
+    for (const f of frontend.fetches) {
+      sections.push(`| ${f.method} | ${f.url} |`);
+    }
+  } else {
+    sections.push('_No fetch calls found._');
+  }
+
+  // 9. Frontend Storage Keys
+  sections.push('\n## Frontend Storage Keys (auto)\n');
+  if (frontend.storageKeys.length > 0) {
+    sections.push('| Operation | Key |');
+    sections.push('|-----------|-----|');
+    for (const s of frontend.storageKeys) {
+      sections.push(`| ${s.operation} | ${s.key} |`);
+    }
+  } else {
+    sections.push('_No sessionStorage usage found._');
+  }
+
+  // Warnings (if any)
+  if (warnings.length > 0) {
+    sections.push('\n---');
+    sections.push('_Some sources could not be read. Sections above may be incomplete._');
+  }
+
+  return sections.join('\n');
+}
+
+module.exports = { formatSections };

package/src/agents/merger.js

@@ -0,0 +1,70 @@
+const fs = require('fs');
+
+const START_MARKER = '<!-- CARTO:AUTO:START -->';
+const END_MARKER = '<!-- CARTO:AUTO:END -->';
+
+/**
+ * Safely writes auto-generated content into AGENTS.md between markers.
+ * Never touches anything outside the markers.
+ * Uses atomic write (write to .tmp, then rename) to prevent corruption.
+ *
+ * Cases:
+ *   1. File does not exist → create with markers + content
+ *   2. File exists, no markers → append markers + content at end
+ *   3. File exists, markers reversed (END before START) → treat as corrupted, append
+ *   4. File exists, valid markers → replace ONLY between markers
+ */
+function mergeIntoAgentsMd(agentsPath, autoContent) {
+  const markerBlock = `${START_MARKER}\n${autoContent}\n${END_MARKER}`;
+
+  // Case 1: File does not exist
+  if (!fs.existsSync(agentsPath)) {
+    const tmpPath = agentsPath + '.tmp';
+    fs.writeFileSync(tmpPath, markerBlock + '\n', 'utf-8');
+    fs.renameSync(tmpPath, agentsPath);
+    return;
+  }
+
+  let existing;
+  try {
+    existing = fs.readFileSync(agentsPath, 'utf-8');
+  } catch (err) {
+    console.error(`[CARTO] Error reading AGENTS.md: ${err.message}`);
+    return;
+  }
+
+  const startIdx = existing.indexOf(START_MARKER);
+  const endIdx = existing.indexOf(END_MARKER);
+
+  // Case 2: No markers found
+  if (startIdx === -1 || endIdx === -1) {
+    const separator = existing.endsWith('\n') ? '\n' : '\n\n';
+    const tmpPath = agentsPath + '.tmp';
+    fs.writeFileSync(tmpPath, existing + separator + markerBlock + '\n', 'utf-8');
+    fs.renameSync(tmpPath, agentsPath);
+    return;
+  }
+
+  // Case 3: Markers reversed or overlapping — treat as corrupted
+  if (startIdx >= endIdx) {
+    const separator = existing.endsWith('\n') ? '\n' : '\n\n';
+    const tmpPath = agentsPath + '.tmp';
+    fs.writeFileSync(tmpPath, existing + separator + markerBlock + '\n', 'utf-8');
+    fs.renameSync(tmpPath, agentsPath);
+    console.warn('[CARTO] Warning: markers were reversed/corrupted — appended fresh marker block');
+    return;
+  }
+
+  // Case 4: Valid markers — replace between them
+  const before = existing.substring(0, startIdx);
+  const after = existing.substring(endIdx + END_MARKER.length);
+  try {
+    const tmpPath = agentsPath + '.tmp';
+    fs.writeFileSync(tmpPath, before + markerBlock + after, 'utf-8');
+    fs.renameSync(tmpPath, agentsPath);
+  } catch (err) {
+    console.error(`[CARTO] Error writing AGENTS.md: ${err.message}`);
+  }
+}
+
+module.exports = { mergeIntoAgentsMd, START_MARKER, END_MARKER };

package/src/cli/index.js

@@ -0,0 +1,43 @@
+#!/usr/bin/env node
+
+const command = process.argv[2];
+
+function printUsage() {
+  console.log(`
+Usage: carto <command>
+
+Commands:
+  init     Detect project, write .carto/config.json, run first sync
+  watch    Read .carto/config.json, start file watcher
+  sync     Read .carto/config.json, run one sync, exit
+
+Options:
+  --help, -h   Show this help message
+`);
+}
+
+if (!command || command === '--help' || command === '-h') {
+  printUsage();
+  process.exit(0);
+}
+
+if (command === 'init') {
+  require('./init').run(process.cwd()).catch(err => {
+    console.error(`[CARTO] Fatal error: ${err.message}`);
+    process.exit(1);
+  });
+} else if (command === 'watch') {
+  require('./watch').run(process.cwd()).catch(err => {
+    console.error(`[CARTO] Fatal error: ${err.message}`);
+    process.exit(1);
+  });
+} else if (command === 'sync') {
+  require('./sync').run(process.cwd()).catch(err => {
+    console.error(`[CARTO] Fatal error: ${err.message}`);
+    process.exit(1);
+  });
+} else {
+  console.error(`[CARTO] Unknown command: ${command}`);
+  printUsage();
+  process.exit(1);
+}

package/src/cli/init.js

@@ -0,0 +1,80 @@
+const fs = require('fs');
+const path = require('path');
+const { detectFramework } = require('../detector/framework');
+const { discoverFiles } = require('../detector/files');
+const { parseCartoIgnore } = require('../security/ignore');
+const { runFullSync } = require('../sync');
+
+async function run(projectRoot) {
+  console.log('[CARTO] Detecting project...');
+
+  const detection = detectFramework(projectRoot);
+  console.log(`[CARTO] Detected: ${detection.framework} (${detection.language})`);
+
+  const isIgnored = parseCartoIgnore(projectRoot);
+  const files = discoverFiles(projectRoot, detection.framework, isIgnored);
+
+  // Count files for reporting
+  const pyCount = files.routeFiles.filter(f => f.endsWith('.py')).length;
+  const jsCount = files.routeFiles.filter(f => /\.(js|ts|jsx|tsx)$/.test(f)).length;
+  const htmlCount = files.frontendFiles.length;
+
+  const parts = [];
+  if (pyCount > 0) parts.push(`${pyCount} Python files`);
+  if (jsCount > 0) parts.push(`${jsCount} JS/TS files`);
+  if (htmlCount > 0) parts.push(`${htmlCount} HTML files`);
+  console.log(`[CARTO] Found ${parts.join(', ') || '0 files'}`);
+
+  // Make paths relative for config storage
+  const relRouteFiles = files.routeFiles.map(f => path.relative(projectRoot, f));
+  const relModelFiles = files.modelFiles.map(f => path.relative(projectRoot, f));
+  const relFrontendFiles = files.frontendFiles.map(f => path.relative(projectRoot, f));
+
+  // Write .carto/config.json
+  const cartoDir = path.join(projectRoot, '.carto');
+  if (!fs.existsSync(cartoDir)) {
+    fs.mkdirSync(cartoDir, { recursive: true });
+  }
+
+  const config = {
+    version: '1',
+    framework: detection.framework,
+    language: detection.language,
+    watch: {
+      routeFiles: relRouteFiles,
+      modelFiles: relModelFiles,
+      frontendFiles: relFrontendFiles
+    },
+    output: 'AGENTS.md',
+    generated: new Date().toISOString()
+  };
+
+  fs.writeFileSync(
+    path.join(cartoDir, 'config.json'),
+    JSON.stringify(config, null, 2) + '\n',
+    'utf-8'
+  );
+
+  // Run first sync
+  const syncConfig = resolveConfig(projectRoot, config);
+  await runFullSync(syncConfig);
+
+  console.log('[CARTO] AGENTS.md generated. Run "carto watch" to keep it live.');
+}
+
+/**
+ * Resolves relative paths in config to absolute paths.
+ */
+function resolveConfig(projectRoot, config) {
+  return {
+    watch: {
+      routeFiles: (config.watch.routeFiles || []).map(f => path.resolve(projectRoot, f)),
+      modelFiles: (config.watch.modelFiles || []).map(f => path.resolve(projectRoot, f)),
+      frontendFiles: (config.watch.frontendFiles || []).map(f => path.resolve(projectRoot, f))
+    },
+    output: path.resolve(projectRoot, config.output),
+    projectRoot
+  };
+}
+
+module.exports = { run, resolveConfig };

package/src/cli/sync.js

@@ -0,0 +1,27 @@
+const fs = require('fs');
+const path = require('path');
+const { runFullSync } = require('../sync');
+const { resolveConfig } = require('./init');
+
+async function run(projectRoot) {
+  const configPath = path.join(projectRoot, '.carto', 'config.json');
+
+  if (!fs.existsSync(configPath)) {
+    console.error('[CARTO] Run "carto init" first.');
+    process.exit(1);
+  }
+
+  let config;
+  try {
+    config = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
+  } catch (err) {
+    console.error(`[CARTO] Error reading .carto/config.json: ${err.message}`);
+    process.exit(1);
+  }
+
+  const resolved = resolveConfig(projectRoot, config);
+  await runFullSync(resolved);
+  console.log('[CARTO] Sync complete.');
+}
+
+module.exports = { run };

package/src/cli/watch.js

@@ -0,0 +1,52 @@
+const fs = require('fs');
+const path = require('path');
+const { startWatcher } = require('../watcher/watch');
+const { runFullSync } = require('../sync');
+const { resolveConfig } = require('./init');
+
+async function run(projectRoot) {
+  const configPath = path.join(projectRoot, '.carto', 'config.json');
+
+  if (!fs.existsSync(configPath)) {
+    console.error('[CARTO] Run "carto init" first.');
+    process.exit(1);
+  }
+
+  let config;
+  try {
+    config = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
+  } catch (err) {
+    console.error(`[CARTO] Error reading .carto/config.json: ${err.message}`);
+    process.exit(1);
+  }
+
+  const resolved = resolveConfig(projectRoot, config);
+
+  // Run full sync once on startup
+  console.log('[CARTO] Starting initial sync...');
+  await runFullSync(resolved);
+  console.log('[CARTO] Initial sync complete');
+
+  // Collect all watch paths (deduplicated)
+  const allFiles = new Set([
+    ...resolved.watch.routeFiles,
+    ...resolved.watch.modelFiles,
+    ...resolved.watch.frontendFiles
+  ]);
+  const watchPaths = [...allFiles];
+
+  startWatcher(watchPaths, async (changedFile) => {
+    await runFullSync(resolved);
+    const timestamp = new Date().toISOString();
+    const filename = path.basename(changedFile);
+    console.log(`[CARTO] ${filename} updated → AGENTS.md synced — ${timestamp}`);
+  });
+
+  console.log('[CARTO] Watching files...');
+  for (const p of watchPaths) {
+    console.log(`  → ${p}`);
+  }
+  console.log(`  → Output: ${resolved.output}`);
+}
+
+module.exports = { run };

package/src/detector/files.js

@@ -0,0 +1,92 @@
+const fs = require('fs');
+const path = require('path');
+
+const MAX_FILES_PER_CATEGORY = 50;
+
+const PYTHON_IGNORE = new Set(['__pycache__', '.venv', 'venv', 'migrations', 'node_modules', '.git', '.carto']);
+const JS_IGNORE = new Set(['node_modules', '.git', 'dist', 'build', '.carto']);
+const HTML_IGNORE = new Set(['node_modules', '.git', '.carto']);
+
+/**
+ * discoverFiles(projectRoot, framework, isIgnored) → { routeFiles, modelFiles, frontendFiles }
+ *
+ * isIgnored is an optional function (filePath) → boolean from the .cartoignore parser.
+ */
+function discoverFiles(projectRoot, framework, isIgnored) {
+  const ignoreFn = isIgnored || (() => false);
+
+  if (['fastapi', 'django', 'flask', 'python-generic'].includes(framework)) {
+    const pyFiles = findFilesRecursive(projectRoot, ['.py'], PYTHON_IGNORE, ignoreFn)
+      .filter(f => {
+        const base = path.basename(f);
+        return !base.startsWith('test_') && !base.endsWith('_test.py');
+      });
+    const htmlFiles = findFilesRecursive(projectRoot, ['.html'], HTML_IGNORE, ignoreFn);
+
+    return {
+      routeFiles: cap(pyFiles),
+      modelFiles: cap(pyFiles),
+      frontendFiles: cap(htmlFiles)
+    };
+  }
+
+  if (['express', 'nextjs', 'react', 'node-generic'].includes(framework)) {
+    const jsFiles = findFilesRecursive(projectRoot, ['.js', '.ts', '.jsx', '.tsx'], JS_IGNORE, ignoreFn)
+      .filter(f => {
+        const base = path.basename(f);
+        return !base.includes('.test.') && !base.includes('.spec.');
+      });
+    const htmlFiles = findFilesRecursive(projectRoot, ['.html'], HTML_IGNORE, ignoreFn);
+
+    return {
+      routeFiles: cap(jsFiles),
+      modelFiles: cap(jsFiles),
+      frontendFiles: cap(htmlFiles)
+    };
+  }
+
+  // Unknown framework — best effort
+  const allCode = findFilesRecursive(projectRoot, ['.py', '.js', '.ts'], new Set(['node_modules', '.git', '__pycache__', '.venv', 'venv', '.carto']), ignoreFn);
+  const htmlFiles = findFilesRecursive(projectRoot, ['.html'], HTML_IGNORE, ignoreFn);
+
+  return {
+    routeFiles: cap(allCode),
+    modelFiles: cap(allCode),
+    frontendFiles: cap(htmlFiles)
+  };
+}
+
+function findFilesRecursive(dir, extensions, ignoreDirs, isIgnored, results = []) {
+  let items;
+  try {
+    items = fs.readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return results;
+  }
+
+  for (const item of items) {
+    if (ignoreDirs.has(item.name)) continue;
+
+    const fullPath = path.join(dir, item.name);
+
+    if (item.isDirectory()) {
+      findFilesRecursive(fullPath, extensions, ignoreDirs, isIgnored, results);
+    } else if (item.isFile()) {
+      const ext = path.extname(item.name).toLowerCase();
+      if (extensions.includes(ext) && !isIgnored(fullPath)) {
+        results.push(fullPath);
+      }
+    }
+  }
+  return results;
+}
+
+function cap(files) {
+  if (files.length > MAX_FILES_PER_CATEGORY) {
+    console.warn(`[CARTO] Warning: Found ${files.length} files, capping at ${MAX_FILES_PER_CATEGORY}`);
+    return files.slice(0, MAX_FILES_PER_CATEGORY);
+  }
+  return files;
+}
+
+module.exports = { discoverFiles };

package/src/detector/framework.js

@@ -0,0 +1,117 @@
+const fs = require('fs');
+const path = require('path');
+
+const IGNORE_DIRS = new Set(['node_modules', '.git', '__pycache__', '.venv', 'venv', '.idea', '.vscode', '.carto']);
+
+/**
+ * detectFramework(projectRoot) → { framework, language, confidence }
+ *
+ * Search order (recursive up to 3 levels deep):
+ * 1. requirements.txt → fastapi / django / flask / python-generic
+ * 2. package.json → nextjs / express / react / node-generic
+ * 3. pyproject.toml → same logic as requirements.txt
+ * 4. Nothing found → { framework: 'unknown', language: 'unknown' }
+ *
+ * Returns first match found. Does not combine multiple detections.
+ */
+function detectFramework(projectRoot) {
+  // Search for files up to 3 levels deep
+  const candidates = findFile(projectRoot, ['requirements.txt', 'package.json', 'pyproject.toml'], 3);
+
+  // 1. Check requirements.txt
+  const reqFile = candidates.find(f => path.basename(f) === 'requirements.txt');
+  if (reqFile) {
+    const result = detectFromPythonDeps(reqFile);
+    if (result) return result;
+  }
+
+  // 2. Check package.json
+  const pkgFile = candidates.find(f => path.basename(f) === 'package.json');
+  if (pkgFile) {
+    const result = detectFromPackageJson(pkgFile);
+    if (result) return result;
+  }
+
+  // 3. Check pyproject.toml
+  const pyprojectFile = candidates.find(f => path.basename(f) === 'pyproject.toml');
+  if (pyprojectFile) {
+    const result = detectFromPythonDeps(pyprojectFile);
+    if (result) return result;
+  }
+
+  return { framework: 'unknown', language: 'unknown', confidence: 'none' };
+}
+
+/**
+ * Recursively find files by name up to maxDepth levels.
+ */
+function findFile(dir, fileNames, maxDepth, currentDepth = 0) {
+  const results = [];
+  if (currentDepth > maxDepth) return results;
+
+  let items;
+  try {
+    items = fs.readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return results;
+  }
+
+  for (const item of items) {
+    if (IGNORE_DIRS.has(item.name)) continue;
+
+    const fullPath = path.join(dir, item.name);
+    if (item.isFile() && fileNames.includes(item.name)) {
+      results.push(fullPath);
+    } else if (item.isDirectory() && currentDepth < maxDepth) {
+      results.push(...findFile(fullPath, fileNames, maxDepth, currentDepth + 1));
+    }
+  }
+  return results;
+}
+
+function detectFromPythonDeps(filePath) {
+  let content;
+  try {
+    content = fs.readFileSync(filePath, 'utf-8').toLowerCase();
+  } catch {
+    return null;
+  }
+
+  if (content.includes('fastapi')) {
+    return { framework: 'fastapi', language: 'python', confidence: 'high' };
+  }
+  if (content.includes('django')) {
+    return { framework: 'django', language: 'python', confidence: 'high' };
+  }
+  if (content.includes('flask')) {
+    return { framework: 'flask', language: 'python', confidence: 'high' };
+  }
+  if (content.includes('pydantic')) {
+    return { framework: 'python-generic', language: 'python', confidence: 'medium' };
+  }
+  return null;
+}
+
+function detectFromPackageJson(filePath) {
+  let pkg;
+  try {
+    pkg = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
+  } catch {
+    return null;
+  }
+
+  const deps = Object.assign({}, pkg.dependencies || {}, pkg.devDependencies || {});
+
+  if (deps['next']) {
+    return { framework: 'nextjs', language: 'javascript', confidence: 'high' };
+  }
+  if (deps['express']) {
+    return { framework: 'express', language: 'javascript', confidence: 'high' };
+  }
+  if (deps['react']) {
+    return { framework: 'react', language: 'javascript', confidence: 'high' };
+  }
+  return { framework: 'node-generic', language: 'javascript', confidence: 'medium' };
+}
+
+module.exports = { detectFramework };