carto-md 2.0.5 → 2.0.7

package/src/store/sync-v2.js

@@ -24,7 +24,10 @@ const IGNORE_DIRS = new Set([
   'dist', '.next', '.turbo', 'build', 'coverage', '.carto',
   'out', '.cache', 'generated', '__generated__',
   'storybook-static', 'public', 'static',
-  'tmp-bench', 'vendor', 'third_party', '.yarn'
+  'tmp-bench', 'vendor', 'third_party', '.yarn',
+  // Test directories — ported from V1 detector/files.js
+  'test', 'tests', '__tests__', 'e2e', 'playwright',
+  'cypress', 'fixtures', 'mocks', '__mocks__'
 ]);
 
 const CODE_EXTS = new Set([
@@ -34,6 +37,36 @@ const CODE_EXTS = new Set([
   '.swift', '.kt'
 ]);
 
+const JS_LIKE_EXTS = new Set(['.js', '.jsx', '.ts', '.tsx', '.mjs', '.cjs']);
+
+/**
+ * isTestFile(relPath) → true if the file is a test/spec/stories file
+ * Ported from V1 detector/files.js exclusion patterns.
+ *   R:      test_*, test-*, *_test.r (case-insensitive)
+ *   Python: test_*.py, *_test.py
+ *   JS/TS:  *.test.*, *.spec.*, *.stories.*
+ * Path-based directory exclusions (test/, tests/, __tests__/, etc.)
+ * are handled by IGNORE_DIRS during the walk — see above.
+ */
+function isTestFile(relPath) {
+  const base = path.basename(relPath);
+  const lbase = base.toLowerCase();
+  const ext = path.extname(base).toLowerCase();
+
+  if (ext === '.r') {
+    return lbase.startsWith('test_') || lbase.startsWith('test-') || lbase.endsWith('_test.r');
+  }
+  if (ext === '.py') {
+    return lbase.startsWith('test_') || lbase.endsWith('_test.py');
+  }
+  if (JS_LIKE_EXTS.has(ext)) {
+    if (lbase.includes('.test.') || lbase.includes('.spec.') || lbase.includes('.stories.')) {
+      return true;
+    }
+  }
+  return false;
+}
+
 /**
  * discoverFiles(projectRoot)
  * Recursively walks the project tree. No file cap. Respects ignore dirs + .cartoignore.
@@ -55,7 +88,7 @@ function discoverFiles(projectRoot) {
         walk(full);
       } else if (entry.isFile()) {
         const ext = path.extname(entry.name).toLowerCase();
-        if (CODE_EXTS.has(ext) && !isIgnored(rel)) {
+        if (CODE_EXTS.has(ext) && !isIgnored(rel) && !isTestFile(rel)) {
           results.push(rel);
         }
       }
@@ -419,8 +452,8 @@ async function runSyncV2(config) {
     }
   }
 
-  // 8. Generate outputs (only if files changed)
-  if (toProcess.length > 0) {
+  // 8. Generate outputs (only if files changed AND output not suppressed)
+  if (toProcess.length > 0 && config.output !== null && config.output !== false) {
     await generateOutputs(store, config, projectRoot, store.getImportGraph());
   }
 

package/src/cache/file-hash.js

@@ -1,84 +0,0 @@
-'use strict';
-
-const fs = require('fs');
-const path = require('path');
-const crypto = require('crypto');
-
-function getHashPath(projectRoot) {
-  return path.join(projectRoot, '.carto', 'hashes.json');
-}
-
-function loadHashes(projectRoot) {
-  try {
-    const raw = fs.readFileSync(getHashPath(projectRoot), 'utf-8');
-    return JSON.parse(raw);
-  } catch {
-    return {};
-  }
-}
-
-function saveHashes(projectRoot, hashes) {
-  const hashPath = getHashPath(projectRoot);
-  const tmp = hashPath + '.tmp';
-  try {
-    fs.writeFileSync(tmp, JSON.stringify(hashes, null, 2), 'utf-8');
-    fs.renameSync(tmp, hashPath);
-  } catch {}
-}
-
-function hashContent(content) {
-  return crypto.createHash('sha1').update(content).digest('hex');
-}
-
-/**
- * computeChangedFiles(filePaths, storedHashes, projectRoot)
- * Returns { changed: string[], unchanged: string[], hashes: object }
- * changed = files whose content hash differs from stored
- * unchanged = files whose hash matches — can skip re-parsing
- */
-function computeChangedFiles(filePaths, storedHashes, projectRoot) {
-  const changed = [];
-  const unchanged = [];
-  const newHashes = { ...storedHashes };
-
-  for (const filePath of filePaths) {
-    const relPath = path.relative(projectRoot, filePath);
-    let content;
-    try {
-      content = fs.readFileSync(filePath, 'utf-8');
-    } catch {
-      continue;
-    }
-    const hash = hashContent(content);
-    if (storedHashes[relPath] === hash) {
-      unchanged.push(filePath);
-    } else {
-      changed.push(filePath);
-      newHashes[relPath] = hash;
-    }
-  }
-
-  return { changed, unchanged, hashes: newHashes };
-}
-
-/**
- * updateFileHash(projectRoot, relPath, content)
- * Updates the hash for a single file after incremental re-index.
- */
-function updateFileHash(projectRoot, relPath, content) {
-  const hashes = loadHashes(projectRoot);
-  hashes[relPath] = hashContent(content);
-  saveHashes(projectRoot, hashes);
-}
-
-/**
- * removeFileHash(projectRoot, relPath)
- * Removes hash entry when a file is deleted.
- */
-function removeFileHash(projectRoot, relPath) {
-  const hashes = loadHashes(projectRoot);
-  delete hashes[relPath];
-  saveHashes(projectRoot, hashes);
-}
-
-module.exports = { loadHashes, saveHashes, hashContent, computeChangedFiles, updateFileHash, removeFileHash };

package/src/cache/graph-cache.js

@@ -1,77 +0,0 @@
-'use strict';
-
-const fs = require('fs');
-const path = require('path');
-
-function getCachePath(projectRoot) {
-  return path.join(projectRoot, '.carto', 'graph-cache.json');
-}
-
-/**
- * loadGraphCache(projectRoot)
- * Returns the full persisted graph or null if not found / corrupt.
- *
- * Cache shape:
- * {
- *   version: '2',
- *   generated: ISO string,
- *   fileData: {
- *     [relPath]: { routes, models, functions, envVars, dbTables, fetches, storageKeys, imports }
- *   },
- *   importGraph: { [relPath]: [relPath, ...] },
- *   routesByFile: { [relPath]: ['METHOD /path', ...] },
- *   domains: { [domain]: { files, routes, models, functions, envVars, dbTables } },
- *   highImpact: [{ file, dependents }],
- *   entryPoints: [relPath, ...],
- *   stack: [...],
- *   meta: { totalFiles, totalRoutes, totalImportEdges, lastIndexed, indexDuration }
- * }
- */
-function loadGraphCache(projectRoot) {
-  try {
-    const raw = fs.readFileSync(getCachePath(projectRoot), 'utf-8');
-    const parsed = JSON.parse(raw);
-    if (parsed.version !== '2') return null;
-    return parsed;
-  } catch {
-    return null;
-  }
-}
-
-function saveGraphCache(projectRoot, cache) {
-  const cachePath = getCachePath(projectRoot);
-  const tmp = cachePath + '.tmp';
-  try {
-    fs.mkdirSync(path.dirname(cachePath), { recursive: true });
-    fs.writeFileSync(tmp, JSON.stringify(cache, null, 2) + '\n', 'utf-8');
-    fs.renameSync(tmp, cachePath);
-  } catch (err) {
-    console.warn(`[CARTO] Warning: Could not save graph cache — ${err.message}`);
-  }
-}
-
-/**
- * buildEmptyCache() — starting point for a fresh index
- */
-function buildEmptyCache() {
-  return {
-    version: '2',
-    generated: new Date().toISOString(),
-    fileData: {},
-    importGraph: {},
-    routesByFile: {},
-    domains: {},
-    highImpact: [],
-    entryPoints: [],
-    stack: [],
-    meta: {
-      totalFiles: 0,
-      totalRoutes: 0,
-      totalImportEdges: 0,
-      lastIndexed: null,
-      indexDuration: 0
-    }
-  };
-}
-
-module.exports = { loadGraphCache, saveGraphCache, buildEmptyCache, getCachePath };

package/src/detector/files.js

@@ -1,289 +0,0 @@
-const fs = require('fs');
-const path = require('path');
-
-const MAX_ROUTE_FILES = 150;    // all route files up to this — routes are never crowded out
-const MAX_MODEL_FILES = 50;     // all model/schema files up to this
-const MAX_UTILITY_FILES = 100;  // top N utilities by import count
-
-const PYTHON_IGNORE = new Set(['__pycache__', '.venv', 'venv', 'migrations', 'node_modules', '.git', '.carto']);
-const JS_IGNORE = new Set(['node_modules', '.git', 'dist', 'build', '.carto', '.next', '.turbo', 'coverage', 'out', '.cache', 'generated', '__generated__', 'storybook-static', 'public', 'static', 'playwright', 'e2e', '__tests__', 'fixtures', 'mocks', '__mocks__', 'cypress']);
-const HTML_IGNORE = new Set(['node_modules', '.git', '.carto']);
-const R_IGNORE = new Set(['.Rhistory', '.RData', 'packrat', 'renv', 'node_modules', '.git', '__pycache__', '.carto']);
-
-/**
- * discoverFiles(projectRoot, framework, isIgnored, secondaryFramework) → { routeFiles, modelFiles, frontendFiles }
- */
-function discoverFiles(projectRoot, framework, isIgnored, secondaryFramework) {
-  const ignoreFn = isIgnored || (() => false);
-
-  const primary = discoverForFramework(projectRoot, framework, ignoreFn);
-
-  if (secondaryFramework) {
-    const secondary = discoverForFramework(projectRoot, secondaryFramework, ignoreFn);
-    const routeFiles = [...new Set([...primary.routeFiles, ...secondary.routeFiles])];
-    const modelFiles = [...new Set([...primary.modelFiles, ...secondary.modelFiles])];
-    const frontendFiles = [...new Set([...primary.frontendFiles, ...secondary.frontendFiles])];
-    return { routeFiles, modelFiles, frontendFiles };
-  }
-
-  return primary;
-}
-
-function discoverForFramework(projectRoot, framework, ignoreFn) {
-  if (['fastapi', 'django', 'flask', 'python-generic'].includes(framework)) {
-    const pyFiles = findFilesRecursive(projectRoot, ['.py'], PYTHON_IGNORE, ignoreFn)
-      .filter(f => {
-        const base = path.basename(f);
-        return !base.startsWith('test_') && !base.endsWith('_test.py');
-      });
-    const htmlFiles = findFilesRecursive(projectRoot, ['.html'], HTML_IGNORE, ignoreFn);
-
-    if (pyFiles.length <= MAX_ROUTE_FILES + MAX_MODEL_FILES + MAX_UTILITY_FILES) {
-      return { routeFiles: pyFiles, modelFiles: pyFiles, frontendFiles: htmlFiles };
-    }
-
-    return smartSelect(pyFiles, htmlFiles, projectRoot);
-  }
-
-  if (['express', 'nextjs', 'react', 'node-generic'].includes(framework)) {
-    const jsFiles = findFilesRecursive(projectRoot, ['.js', '.ts', '.jsx', '.tsx', '.prisma'], JS_IGNORE, ignoreFn)
-      .filter(f => {
-        const base = path.basename(f);
-        const rel = f.toLowerCase().replace(/\\/g, '/');
-        return !base.includes('.test.') &&
-               !base.includes('.spec.') &&
-               !base.includes('.stories.') &&
-               !rel.includes('/test/') &&
-               !rel.includes('/tests/') &&
-               !rel.includes('/e2e/') &&
-               !rel.includes('/playwright/') &&
-               !rel.includes('/__tests__/') &&
-               !rel.includes('/fixtures/');
-      });
-    const htmlFiles = findFilesRecursive(projectRoot, ['.html'], HTML_IGNORE, ignoreFn);
-
-    if (jsFiles.length <= MAX_ROUTE_FILES + MAX_MODEL_FILES + MAX_UTILITY_FILES) {
-      return { routeFiles: jsFiles, modelFiles: jsFiles, frontendFiles: htmlFiles };
-    }
-
-    return smartSelect(jsFiles, htmlFiles, projectRoot);
-  }
-
-  if (['plumber', 'shiny', 'r-generic'].includes(framework)) {
-    const rFiles = findFilesRecursive(projectRoot, ['.r'], R_IGNORE, ignoreFn)
-      .filter(f => {
-        const lbase = path.basename(f).toLowerCase();
-        return !lbase.startsWith('test_') && !lbase.startsWith('test-') && !lbase.endsWith('_test.r');
-      });
-
-    if (rFiles.length <= MAX_ROUTE_FILES + MAX_MODEL_FILES + MAX_UTILITY_FILES) {
-      return { routeFiles: rFiles, modelFiles: rFiles, frontendFiles: [] };
-    }
-
-    return smartSelect(rFiles, [], projectRoot);
-  }
-
-  // Unknown framework
-  const allCode = findFilesRecursive(projectRoot, ['.py', '.js', '.ts'], new Set(['node_modules', '.git', '__pycache__', '.venv', 'venv', '.carto']), ignoreFn);
-  const htmlFiles = findFilesRecursive(projectRoot, ['.html'], HTML_IGNORE, ignoreFn);
-
-  if (allCode.length <= MAX_ROUTE_FILES + MAX_MODEL_FILES + MAX_UTILITY_FILES) {
-    return { routeFiles: allCode, modelFiles: allCode, frontendFiles: htmlFiles };
-  }
-
-  return smartSelect(allCode, htmlFiles, projectRoot);
-}
-
-function smartSelect(allFiles, htmlFiles, projectRoot) {
-  console.warn(`[CARTO] Warning: Found ${allFiles.length} files, selecting by tier`);
-
-  const routeCandidates = [];
-  const modelCandidates = [];
-  const otherFiles = [];
-
-  for (const f of allFiles) {
-    if (isRouteFile(f)) {
-      routeCandidates.push(f);
-    } else if (isModelFile(f)) {
-      modelCandidates.push(f);
-    } else {
-      otherFiles.push(f);
-    }
-  }
-
-  // Tier 1 — routes: all of them up to MAX_ROUTE_FILES
-  // Split tRPC routers and REST routes, tRPC gets up to 50 slots
-  const trpcRouters = routeCandidates.filter(f => f.includes('/routers/'));
-  const restRoutes = routeCandidates.filter(f => !f.includes('/routers/'));
-
-  trpcRouters.sort((a, b) => scoreRoute(b) - scoreRoute(a));
-  restRoutes.sort((a, b) => scoreRoute(b) - scoreRoute(a));
-
-  const trpcSelected = trpcRouters.slice(0, Math.min(trpcRouters.length, 50));
-  const restSelected = restRoutes.slice(0, Math.min(restRoutes.length, MAX_ROUTE_FILES - trpcSelected.length));
-  const selectedRoutes = [...trpcSelected, ...restSelected];
-
-  // Tier 2 — models: all of them up to MAX_MODEL_FILES
-  modelCandidates.sort((a, b) => scoreModel(b) - scoreModel(a));
-  const selectedModels = modelCandidates.slice(0, MAX_MODEL_FILES);
-
-  // Tier 3 — utilities: top N by import count
-  const importCounts = countImportReferences(allFiles, projectRoot);
-  otherFiles.sort((a, b) => {
-    const relA = path.relative(projectRoot, a);
-    const relB = path.relative(projectRoot, b);
-    return (importCounts[relB] || 0) - (importCounts[relA] || 0);
-  });
-
-  const alreadySelected = new Set([...selectedRoutes, ...selectedModels]);
-  const selectedUtilities = otherFiles
-    .filter(f => !alreadySelected.has(f))
-    .slice(0, MAX_UTILITY_FILES);
-
-  const allSelected = [...new Set([...selectedRoutes, ...selectedModels, ...selectedUtilities])];
-
-  console.warn(`[CARTO] Selected: ${selectedRoutes.length} route files, ${selectedModels.length} model files, ${selectedUtilities.length} utility files`);
-
-  return {
-    routeFiles: allSelected,
-    modelFiles: allSelected,
-    frontendFiles: htmlFiles.slice(0, 10)
-  };
-}
-
-function isRouteFile(filePath) {
-  const p = filePath.toLowerCase().replace(/\\/g, '/');
-  const base = path.basename(p);
-  if (base === 'route.ts' || base === 'route.js' || base === 'routes.ts' || base === 'routes.js') return true;
-  if (p.includes('/api/') || p.includes('/routes/') || p.includes('/route/')) return true;
-  if (p.includes('/routers/')) return true;
-  if (base === 'main.py' || base === 'app.py' || base === 'server.ts' || base === 'server.js') return true;
-  if (p.includes('/pages/api/')) return true;
-  return false;
-}
-
-function isModelFile(filePath) {
-  const p = filePath.toLowerCase().replace(/\\/g, '/');
-  const base = path.basename(p);
-  if (base === 'schema.prisma' || base.endsWith('.prisma')) return true;
-  if (base.startsWith('models.') || base.includes('.model.') || base.includes('.models.')) return true;
-  if (p.includes('/models/') || p.includes('/schemas/') || p.includes('/schema/')) return true;
-  if (base.includes('entity') || base.includes('schema')) return true;
-  return false;
-}
-
-function scoreRoute(filePath) {
-  const p = filePath.toLowerCase().replace(/\\/g, '/');
-  const base = path.basename(p);
-  if (base === 'main.py' || base === 'app.py' || base === 'server.ts' || base === 'server.js') return 10;
-  if (p.includes('/app/api/')) return 9;
-  if (p.includes('/pages/api/')) return 8;
-  if (p.includes('/routers/')) return 9;
-  if (p.includes('/routes/') || p.includes('/api/')) return 7;
-  return 5;
-}
-
-function scoreModel(filePath) {
-  const p = filePath.toLowerCase().replace(/\\/g, '/');
-  const base = path.basename(p);
-  if (base === 'schema.prisma') return 10;
-  if (base.startsWith('models.')) return 9;
-  if (p.includes('/models/')) return 8;
-  if (base.includes('.model.')) return 7;
-  return 5;
-}
-
-/**
- * Quick scan of all files for import/require statements.
- * Returns { 'relative/path': count } of how many files import each path.
- */
-function countImportReferences(allFiles, projectRoot) {
-  const counts = {};
-
-  // Build a set of known relative paths for matching
-  const knownPaths = new Set();
-  for (const f of allFiles) {
-    knownPaths.add(path.relative(projectRoot, f));
-  }
-
-  // Quick regex scan — don't parse AST, just count references
-  const importPattern = /(?:from|import)\s+['"]([^'"]+)['"]|require\s*\(\s*['"]([^'"]+)['"]\s*\)/g;
-
-  for (const filePath of allFiles) {
-    let content;
-    try {
-      content = fs.readFileSync(filePath, 'utf-8');
-    } catch {
-      continue;
-    }
-
-    const fileDir = path.dirname(filePath);
-    let match;
-    importPattern.lastIndex = 0;
-    while ((match = importPattern.exec(content)) !== null) {
-      const importPath = match[1] || match[2];
-      if (!importPath || !importPath.startsWith('.')) continue;
-
-      // Try to resolve to a known file
-      const resolved = tryResolve(importPath, fileDir, projectRoot, knownPaths);
-      if (resolved) {
-        counts[resolved] = (counts[resolved] || 0) + 1;
-      }
-    }
-  }
-
-  return counts;
-}
-
-/**
- * Try to resolve a relative import to a known file path.
- */
-function tryResolve(importPath, fileDir, projectRoot, knownPaths) {
-  const base = path.resolve(fileDir, importPath);
-  const rel = path.relative(projectRoot, base);
-
-  // Try exact
-  if (knownPaths.has(rel)) return rel;
-
-  // Try extensions
-  const extensions = ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.py'];
-  for (const ext of extensions) {
-    const withExt = rel + ext;
-    if (knownPaths.has(withExt)) return withExt;
-  }
-
-  // Try index files
-  for (const ext of extensions) {
-    const indexFile = path.join(rel, 'index' + ext);
-    if (knownPaths.has(indexFile)) return indexFile;
-  }
-
-  return null;
-}
-
-function findFilesRecursive(dir, extensions, ignoreDirs, isIgnored, results = []) {
-  let items;
-  try {
-    items = fs.readdirSync(dir, { withFileTypes: true });
-  } catch {
-    return results;
-  }
-
-  for (const item of items) {
-    if (ignoreDirs.has(item.name)) continue;
-
-    const fullPath = path.join(dir, item.name);
-
-    if (item.isDirectory()) {
-      findFilesRecursive(fullPath, extensions, ignoreDirs, isIgnored, results);
-    } else if (item.isFile()) {
-      const ext = path.extname(item.name).toLowerCase();
-      if (extensions.includes(ext) && !isIgnored(fullPath)) {
-        results.push(fullPath);
-      }
-    }
-  }
-  return results;
-}
-
-module.exports = { discoverFiles };