carto-md 2.0.7 → 2.0.9

package/src/store/sync-v2.js

@@ -9,13 +9,16 @@ const { detectChangedFiles, hashFile, hashContent } = require('./change-detector
 const { loadLanguagePlugins, getPluginForFile } = require('../extractors/loader');
 const { extractImports } = require('../extractors/imports');
 const { buildStackLine } = require('../extractors/stack');
-const { clusterByDomain } = require('../agents/domains');
+const { clusterByDomain, setDomainMap } = require('../agents/domains');
 const { clusterByGraph } = require('../agents/leiden');
 const { formatSections, formatDomainFile } = require('../agents/formatter');
 const { mergeIntoAgentsMd } = require('../agents/merger');
 const { scanStructure } = require('../agents/scan-structure');
 const { WorkerPool, POOL_SIZE } = require('../engine/worker-pool');
 const { parseCartoIgnore } = require('../security/ignore');
+const { loadCartoConfig, applyAnchors } = require('./config-loader');
+const { buildFromStore: buildBitmap, saveToDisk: saveBitmap } = require('../bitmap/sidecar');
+const { invalidate: invalidateBitmap } = require('../bitmap/index');
 
 const plugins = loadLanguagePlugins();
 
@@ -40,7 +43,7 @@ const CODE_EXTS = new Set([
 const JS_LIKE_EXTS = new Set(['.js', '.jsx', '.ts', '.tsx', '.mjs', '.cjs']);
 
 /**
- * isTestFile(relPath) → true if the file is a test/spec/stories file
+ * isTestFile(relPath) → true if the file is a test or stories file.
  * Ported from V1 detector/files.js exclusion patterns.
  *   R:      test_*, test-*, *_test.r (case-insensitive)
  *   Python: test_*.py, *_test.py
@@ -101,6 +104,14 @@ function discoverFiles(projectRoot) {
 /**
  * extractFile(relPath, projectRoot)
  * Extracts all data from a single file. Returns extraction result or null.
+ *
+ * Instead of just `console.warn`-ing on extractor failures, the
+ * returned result includes an `errors` array of `{ phase, message }` so
+ * the caller can persist them in `extraction_errors`. Phases:
+ *   - 'extract'  → plugin.extract threw (parse error, plugin bug, ...)
+ *   - 'imports'  → extractImports threw
+ * On a plugin.extract throw the file is still indexed with empty
+ * routes/models/etc. — visibility beats silent skipping.
  */
 function extractFile(relPath, projectRoot) {
   const fullPath = path.resolve(projectRoot, relPath);
@@ -110,12 +121,23 @@ function extractFile(relPath, projectRoot) {
   const { content, hash } = fileResult;
   const plugin = getPluginForFile(plugins, fullPath);
 
+  const errors = [];
   let extracted = { routes: [], models: [], functions: [], envVars: [], dbTables: [], fetches: [], storageKeys: [] };
   if (plugin) {
     try {
       extracted = plugin.extract(content, relPath);
+      // Plugins may surface their own internal failures (e.g. Babel parse
+      // errors that they recover from with tree-sitter) via `_errors`.
+      // Promote those into our breadcrumb stream so they show up in
+      // `carto check` rather than only on stderr.
+      if (Array.isArray(extracted._errors) && extracted._errors.length > 0) {
+        for (const e of extracted._errors) {
+          if (e && e.phase && e.message) errors.push({ phase: e.phase, message: e.message });
+        }
+      }
     } catch (err) {
       console.warn(`[CARTO] Extraction error for ${relPath}: ${err.message}`);
+      errors.push({ phase: 'extract', message: err.message || String(err) });
     }
   }
 
@@ -123,7 +145,9 @@ function extractFile(relPath, projectRoot) {
   let imports = [];
   try {
     imports = extractImports(content, fullPath, projectRoot);
-  } catch {}
+  } catch (err) {
+    errors.push({ phase: 'imports', message: err.message || String(err) });
+  }
 
   const stat = fs.statSync(fullPath);
 
@@ -135,7 +159,8 @@ function extractFile(relPath, projectRoot) {
     language: detectLanguage(path.extname(relPath).toLowerCase()),
     content,
     extracted,
-    imports
+    imports,
+    errors
   };
 }
 
@@ -206,7 +231,7 @@ async function runSyncV2(config) {
       const batchTx = store.db.transaction(() => {
         for (const result of results) {
           if (!result) continue;
-          const { relPath, routes, models, functions, envVars, dbTables, imports } = result;
+          const { relPath, routes, models, functions, envVars, dbTables, imports, errors } = result;
 
           const stat = (() => {
             try { return fs.statSync(path.resolve(projectRoot, relPath)); } catch { return null; }
@@ -241,7 +266,8 @@ async function runSyncV2(config) {
             routes: routes || [],
             models: models || [],
             envVars: envVars || [],
-            dbTables: (dbTables || []).map(t => ({ table: t.table || t.table_name || t.name, operation: t.operation }))
+            dbTables: (dbTables || []).map(t => ({ table: t.table || t.table_name || t.name, operation: t.operation })),
+            errors: errors || []
           });
         }
       });
@@ -278,7 +304,8 @@ async function runSyncV2(config) {
           routes: result.extracted.routes || [],
           models: result.extracted.models || [],
           envVars: result.extracted.envVars || [],
-          dbTables: result.extracted.dbTables || []
+          dbTables: result.extracted.dbTables || [],
+          errors: result.errors || []
         });
 
         done++;
@@ -293,6 +320,13 @@ async function runSyncV2(config) {
 
   // 7. Compute reverse deps (only if imports changed)
   if (toProcess.length > 0) {
+    // Repair imports whose `to_file_id` was null because the target
+    // file hadn't been upserted yet during this file's extraction
+    // (alphabetical-order chicken-and-egg). Cheap one-shot UPDATE.
+    const repaired = store.resolveUnresolvedImports();
+    if (repaired > 0) {
+      console.log(`[CARTO] Resolved ${repaired} previously-unresolved imports`);
+    }
     console.log('[CARTO] Computing reverse dependencies...');
     store.computeReverseDeps(5);
   } else {
@@ -302,15 +336,25 @@ async function runSyncV2(config) {
   // 7b. Domain clustering (only if files changed)
   if (toProcess.length > 0) {
     const importGraph = store.getImportGraph();
-
-    // Use graph-based Leiden+CPM clustering for any-repo domain detection
-    // Falls back to keyword clustering if graph is too sparse (< 10 edges)
     const edgeCount = Object.values(importGraph).reduce((s, d) => s + d.length, 0);
+    const fileCount = allFiles.length;
+
+    // 10b — Load carto.config.json for custom domain keywords + anchors
+    const cartoConfig = loadCartoConfig(projectRoot);
+    if (cartoConfig) {
+      // Feed custom keywords into the keyword-based domain map
+      const customKeywords = {};
+      for (const [name, cfg] of Object.entries(cartoConfig.domains)) {
+        if (cfg.keywords && cfg.keywords.length > 0) customKeywords[name] = cfg.keywords;
+      }
+      if (Object.keys(customKeywords).length > 0) setDomainMap(customKeywords);
+    }
 
+    const strategy = selectClusteringStrategy(fileCount, edgeCount);
     let fileAssignments; // Map<filePath, domainName>
 
-    if (edgeCount >= 10) {
-      // Graph-based: works on any repo (vscode, zed, game engines, etc.)
+    if (strategy.method === 'graph') {
+      // Merge default + config keywords for graph naming
       const keywordSeeds = {
         AUTH:          ['auth', 'login', 'session', 'oauth', 'jwt', 'password'],
         PAYMENTS:      ['payment', 'billing', 'stripe', 'invoice', 'subscription'],
@@ -319,97 +363,38 @@ async function runSyncV2(config) {
         EVENTS:        ['webhook', 'event', 'queue', 'job', 'worker', 'cron'],
         NOTIFICATIONS: ['email', 'notification', 'mail', 'sms', 'alert'],
       };
-
-      // Compute graph density to decide clustering strategy
-      // Monorepos have sparse cross-package edges → graph clustering produces
-      // hundreds of micro-communities. Use keyword fallback for sparse graphs.
-      const fileCount = allFiles.length;
-      const density = edgeCount / Math.max(fileCount, 1); // avg edges per file
-
-      let rawAssignments;
-
-      if (density >= 1.5) {
-        // Dense enough for graph clustering (typical app repos)
-        // Scale gamma: larger repos need coarser resolution
-        let gamma = 0.03;
-        if (fileCount > 5000) gamma = 0.08;
-        else if (fileCount > 2000) gamma = 0.05;
-        else if (fileCount > 500) gamma = 0.04;
-
-        rawAssignments = clusterByGraph(importGraph, gamma, keywordSeeds);
-
-        // Merge micro-communities (< minSize files) into CORE
-        const minSize = Math.max(3, Math.floor(fileCount / 100));
-        const domainCounts = new Map();
-        for (const domain of rawAssignments.values()) {
-          domainCounts.set(domain, (domainCounts.get(domain) || 0) + 1);
+      if (cartoConfig) {
+        for (const [name, cfg] of Object.entries(cartoConfig.domains)) {
+          if (cfg.keywords && cfg.keywords.length > 0) {
+            keywordSeeds[name] = [...(keywordSeeds[name] || []), ...cfg.keywords];
+          }
         }
+      }
 
-        fileAssignments = new Map();
-        for (const [fp, domain] of rawAssignments) {
-          const count = domainCounts.get(domain) || 0;
-          fileAssignments.set(fp, count >= minSize ? domain : 'CORE');
-        }
-      } else {
-        // Sparse graph (monorepo, Rust workspace, etc.)
-        // Fall back to keyword-based clustering which works better here
-        const routes = store.getRoutes();
-        const models = store.getModels();
-        const envVars = store.getEnvVars();
-        const routesByFile = buildRoutesByFile(store);
-        const functions = {};
-        const allFilesForDomain = store.getAllFiles();
-        for (const f of allFilesForDomain) {
-          const syms = store.db.prepare(
-            'SELECT name FROM symbols WHERE file_id = ? AND kind = ?'
-          ).all(f.id, 'function').map(r => r.name);
-          if (syms.length > 0) functions[f.path] = syms;
-        }
-        const dbTables = store.db.prepare(`
-          SELECT dt.table_name, dt.operation, f.path as file
-          FROM db_tables dt JOIN files f ON dt.file_id = f.id
-        `).all().map(r => ({ table: r.table_name, operation: r.operation, file: r.file }));
-
-        const domainResult = clusterByDomain({
-          routes, models: models.filter(m => m.name).map(m => ({ ...m, className: m.name })),
-          functions, envVars, dbTables, fileMap: [], routesByFile, importGraph
-        });
+      const rawAssignments = clusterByGraph(importGraph, strategy.gamma, keywordSeeds);
 
-        fileAssignments = new Map();
-        for (const [domainName, cluster] of Object.entries(domainResult)) {
-          for (const fp of (cluster.files || [])) fileAssignments.set(fp, domainName);
-        }
+      // Merge micro-communities (< minSize files) into CORE
+      const domainCounts = new Map();
+      for (const domain of rawAssignments.values()) {
+        domainCounts.set(domain, (domainCounts.get(domain) || 0) + 1);
       }
-    } else {
-      // Sparse graph — fall back to keyword-based clustering
-      const routes = store.getRoutes();
-      const models = store.getModels();
-      const envVars = store.getEnvVars();
-      const routesByFile = buildRoutesByFile(store);
-      const functions = {};
-      const allFilesForDomain = store.getAllFiles();
-      for (const f of allFilesForDomain) {
-        const syms = store.db.prepare(
-          'SELECT name FROM symbols WHERE file_id = ? AND kind = ?'
-        ).all(f.id, 'function').map(r => r.name);
-        if (syms.length > 0) functions[f.path] = syms;
-      }
-      const dbTables = store.db.prepare(`
-        SELECT dt.table_name, dt.operation, f.path as file
-        FROM db_tables dt JOIN files f ON dt.file_id = f.id
-      `).all().map(r => ({ table: r.table_name, operation: r.operation, file: r.file }));
-
-      const domainResult = clusterByDomain({
-        routes, models: models.filter(m => m.name).map(m => ({ ...m, className: m.name })),
-        functions, envVars, dbTables, fileMap: [], routesByFile, importGraph
-      });
-
       fileAssignments = new Map();
-      for (const [domainName, cluster] of Object.entries(domainResult)) {
-        for (const fp of (cluster.files || [])) fileAssignments.set(fp, domainName);
+      for (const [fp, domain] of rawAssignments) {
+        fileAssignments.set(fp, (domainCounts.get(domain) || 0) >= strategy.minSize ? domain : 'CORE');
       }
+    } else {
+      fileAssignments = runKeywordClustering(store, importGraph);
     }
 
+    // 10b — Apply anchor pinning from config
+    if (cartoConfig) applyAnchors(fileAssignments, cartoConfig);
+
+    // Reset domain map to defaults after use (avoid polluting subsequent runs)
+    if (cartoConfig) setDomainMap(null);
+
+    // 10c — Domain stability metric: compare to previous snapshot
+    computeDomainStability(store, fileAssignments);
+
     // Write domain assignments to SQLite
     store.clearDomainAssignments();
     // Group by domain name
@@ -436,6 +421,20 @@ async function runSyncV2(config) {
   store.setMeta('index_duration_ms', String(elapsed));
   store.setMeta('last_full_sync', new Date().toISOString());
 
+  // Extraction error count for fast access by `carto check`,
+  // `carto init` summary, and MCP get_architecture.
+  const extractionErrorCount = store.getExtractionErrorCount();
+  store.setMeta('extraction_error_count', String(extractionErrorCount));
+
+  // Record unavailable grammars so `carto check` and MCP
+  // `get_architecture` can surface language coverage gaps.
+  const { getUnavailableLanguages } = require('../extractors/tree-sitter-parser');
+  const unavailableLangs = getUnavailableLanguages();
+  store.setMeta('unavailable_languages_json', JSON.stringify(unavailableLangs));
+  if (unavailableLangs.length > 0) {
+    console.log(`[CARTO] ⚠️  ${unavailableLangs.length} language grammar${unavailableLangs.length === 1 ? '' : 's'} unavailable (${unavailableLangs.join(', ')}) — using regex fallback`);
+  }
+
   // Detect stack
   if (toProcess.length > 0) {
     const changedContents = [];
@@ -457,11 +456,177 @@ async function runSyncV2(config) {
     await generateOutputs(store, config, projectRoot, store.getImportGraph());
   }
 
+  // Build bitmap sidecar. Always run — the bitmap layer is
+  //    derived + disposable, so we rebuild on every sync to keep
+  //    `.carto/bitmap.bin` aligned with the SQLite source of truth.
+  //    Best-effort: a build/persist failure is logged but never fails the
+  //    sync (MCP tools fall back to SQLite when bitmap is unavailable).
+  //
+  //    The same `sidecar` object is reused below to emit the public
+  //    ANCI v0.1 export (`.carto/anci.{yaml,bin}`) — building it twice
+  //    would be wasteful, so the build/save sequence shares scope.
+  let sidecarForAnci = null;
+  try {
+    const cartoDir = path.join(projectRoot, '.carto');
+    const sidecar = buildBitmap(store);
+    saveBitmap(cartoDir, sidecar);
+    sidecarForAnci = sidecar;
+    // Drop any in-memory cache held by the same Node process — e.g. when
+    // sync runs from inside `carto serve`'s lazy reparse path. Next MCP
+    // query loads the freshly-saved file.
+    invalidateBitmap();
+  } catch (err) {
+    process.stderr.write(
+      `[CARTO] bitmap sidecar build failed (queries will use SQLite): ` +
+      `${err && err.message ? err.message : err}\n`
+    );
+  }
+
+  // Emit ANCI v0.1 public export.
+  // Best-effort: failure logs to stderr, never fails the sync.
+  // Skipped when the bitmap build above failed (no sidecar to emit).
+  if (sidecarForAnci) {
+    try {
+      const cartoDir = path.join(projectRoot, '.carto');
+      const { emitToCartoDir } = require('../anci/emit');
+      emitToCartoDir({ cartoDir, sidecar: sidecarForAnci, store });
+    } catch (err) {
+      process.stderr.write(
+        `[CARTO] ANCI publish failed (anci.{yaml,bin} not written): ` +
+        `${err && err.message ? err.message : err}\n`
+      );
+    }
+  }
+
   console.log(`[CARTO] Indexed ${toProcess.length} files (${cached} cached) in ${elapsed}ms`);
   console.log(`[CARTO] Total: ${allFiles.length} files, ${structure.meta.totalRoutes} routes, ${structure.meta.totalImportEdges} import edges`);
 
+  // Surface extraction failures so users notice them
+  // immediately, not after they ask the AI a question and get bad
+  // answers. Pointed at `carto check` for the full breakdown.
+  if (extractionErrorCount > 0) {
+    console.log(`[CARTO] ⚠️  ${extractionErrorCount} extraction error${extractionErrorCount === 1 ? '' : 's'} (run \`carto check\` for details)`);
+  }
+
   store.close();
-  return { filesProcessed: toProcess.length, totalFiles: allFiles.length, elapsed };
+
+  // store.close() runs `PRAGMA wal_checkpoint(TRUNCATE)` which bumps
+  // carto.db's mtime to *after* we wrote bitmap.bin and anci.{yaml,bin}.
+  // bitmapIsFresh() (src/bitmap/index.js) and `carto inspect` both rely
+  // on the invariant `bitmapMtime >= dbMtime` to decide whether the
+  // sidecar can be reused or has to be rebuilt — so re-stamp the
+  // derived files explicitly past the DB's now-final mtime. Without
+  // this, every cold MCP query after a sync sees the bitmap as "stale"
+  // and pays an unnecessary rebuild from SQLite.
+  //
+  // We stat the DB and stamp bitmap/ANCI to (dbMtime + 1ms) rather than
+  // using `new Date()` because the wall clock + filesystem timestamp
+  // pipeline has sub-millisecond jitter on APFS that can leave the
+  // explicit-set mtime *just* under the DB's real mtime. Reading the
+  // DB's actual mtime and adding a fixed offset is race-free.
+  try {
+    const cartoDir = path.join(projectRoot, '.carto');
+    const dbPath = path.join(cartoDir, 'carto.db');
+    const dbStat = fs.statSync(dbPath);
+    const bumpedMs = dbStat.mtimeMs + 1;
+    const bumpedDate = new Date(bumpedMs);
+    for (const name of ['bitmap.bin', 'anci.bin', 'anci.yaml']) {
+      const p = path.join(cartoDir, name);
+      try { fs.utimesSync(p, bumpedDate, bumpedDate); } catch {}
+    }
+  } catch {}
+
+  return { filesProcessed: toProcess.length, totalFiles: allFiles.length, elapsed, extractionErrorCount };
+}
+
+/**
+ * selectClusteringStrategy(fileCount, edgeCount) → { method, gamma?, minSize? }
+ *
+ * Adaptive clustering strategy selection:
+ *   - <100 files → keyword (kills over-fragmentation for small repos)
+ *   - density <1.5 → keyword (sparse graph / monorepo)
+ *   - else → graph with continuous gamma = min(0.10, 0.02 + 0.02·log10(fileCount/10))
+ *     minSize = clamp(sqrt(fileCount), 5, 20)
+ */
+function selectClusteringStrategy(fileCount, edgeCount) {
+  if (fileCount < 100) return { method: 'keyword' };
+  const density = edgeCount / Math.max(fileCount, 1);
+  if (density < 1.5) return { method: 'keyword' };
+  const gamma = Math.min(0.10, 0.02 + 0.02 * Math.log10(fileCount / 10));
+  const minSize = Math.max(5, Math.min(20, Math.round(Math.sqrt(fileCount))));
+  return { method: 'graph', gamma, minSize };
+}
+
+/**
+ * runKeywordClustering(store, importGraph) → Map<filePath, domainName>
+ * Consolidated keyword-based clustering used for small/sparse repos.
+ */
+function runKeywordClustering(store, importGraph) {
+  const routes = store.getRoutes();
+  const models = store.getModels();
+  const envVars = store.getEnvVars();
+  const routesByFile = buildRoutesByFile(store);
+  const functions = {};
+  const allFilesForDomain = store.getAllFiles();
+  for (const f of allFilesForDomain) {
+    const syms = store.db.prepare(
+      'SELECT name FROM symbols WHERE file_id = ? AND kind = ?'
+    ).all(f.id, 'function').map(r => r.name);
+    if (syms.length > 0) functions[f.path] = syms;
+  }
+  const dbTables = store.db.prepare(`
+    SELECT dt.table_name, dt.operation, f.path as file
+    FROM db_tables dt JOIN files f ON dt.file_id = f.id
+  `).all().map(r => ({ table: r.table_name, operation: r.operation, file: r.file }));
+
+  const domainResult = clusterByDomain({
+    routes, models: models.filter(m => m.name).map(m => ({ ...m, className: m.name })),
+    functions, envVars, dbTables, fileMap: [], routesByFile, importGraph
+  });
+
+  const fileAssignments = new Map();
+  for (const [domainName, cluster] of Object.entries(domainResult)) {
+    for (const fp of (cluster.files || [])) fileAssignments.set(fp, domainName);
+  }
+  return fileAssignments;
+}
+
+/**
+ * computeDomainStability(store, fileAssignments)
+ * Compares current assignments to previous snapshot, stores reassignment %.
+ * Warns if total>=10 and the reassignment rate exceeds 5%.
+ */
+function computeDomainStability(store, fileAssignments) {
+  const prevRaw = store.getMeta('previous_domain_snapshot');
+  const currentSnapshot = {};
+  for (const [fp, domain] of fileAssignments) currentSnapshot[fp] = domain;
+
+  let driftPct = 0;
+  const reassignments = [];
+
+  if (prevRaw) {
+    try {
+      const prev = JSON.parse(prevRaw);
+      const total = fileAssignments.size;
+      let changed = 0;
+      for (const [fp, domain] of fileAssignments) {
+        if (prev[fp] && prev[fp] !== domain) {
+          changed++;
+          if (reassignments.length < 20) {
+            reassignments.push({ file: fp, from: prev[fp], to: domain });
+          }
+        }
+      }
+      driftPct = total > 0 ? (changed / total) * 100 : 0;
+      if (total >= 10 && driftPct > 5) {
+        console.warn(`[CARTO] ⚠️  Domain stability: ${driftPct.toFixed(1)}% files changed domain (${changed}/${total})`);
+      }
+    } catch { /* malformed snapshot — treat as first run */ }
+  }
+
+  store.setMeta('previous_domain_snapshot', JSON.stringify(currentSnapshot));
+  store.setMeta('domain_stability_drift_pct', driftPct.toFixed(2));
+  store.setMeta('last_reassignments_json', JSON.stringify(reassignments));
 }
 
 /**
@@ -618,4 +783,214 @@ function detectLanguage(ext) {
   return map[ext] || 'unknown';
 }
 
-module.exports = { runSyncV2, discoverFiles, extractFile, buildFileDataFromStore, generateOutputs, buildRoutesByFile, detectLanguage };
+/**
+ * syncFiles(projectRoot, paths, opts?) → { reparsed, skipped, removed, elapsed }
+ *
+ * Hot re-parse a small set of files. Used by:
+ *   - The MCP server's lazy mtime check at query time
+ *   - The optional `carto watch` command (incremental save handler)
+ *
+ * Each path is mtime+size-checked against the SQLite row. Unchanged files
+ * are skipped. Changed files are extracted and written, and reverse_deps
+ * are updated incrementally for just the file's neighbors (NOT the whole
+ * graph). No discovery pass, no domain reclustering, no AGENTS.md
+ * regeneration — this is the freshness equivalent of `git status` and
+ * finishes in <50ms for a single file.
+ *
+ * Files that no longer exist on disk are removed from the index.
+ *
+ * Opts:
+ *   store — Optional pre-opened writable SQLiteStore. If omitted, a new
+ *           connection is opened and closed. Pass an existing store when
+ *           batching many calls (e.g. the watcher debounce loop).
+ */
+function syncFiles(projectRoot, paths, opts = {}) {
+  const startTime = Date.now();
+  if (!Array.isArray(paths) || paths.length === 0) {
+    return { reparsed: 0, skipped: 0, removed: 0, elapsed: 0 };
+  }
+
+  const ownsStore = !opts.store;
+  let store = opts.store;
+  if (ownsStore) {
+    store = new SQLiteStore(projectRoot);
+    store.open();
+  }
+
+  let reparsed = 0;
+  let skipped = 0;
+  let removed = 0;
+
+  try {
+    for (const relPath of paths) {
+      if (!relPath || typeof relPath !== 'string') { skipped++; continue; }
+
+      const fullPath = path.resolve(projectRoot, relPath);
+
+      // Stat — if missing, treat as deletion
+      let stat;
+      try {
+        stat = fs.statSync(fullPath);
+      } catch {
+        const existing = store.getFileByPath(relPath);
+        if (existing) {
+          store.removeFile(relPath);
+          updateReverseDepsPartial(store, existing.id);
+          removed++;
+        }
+        continue;
+      }
+
+      const mtime = Math.floor(stat.mtimeMs);
+      const size = stat.size;
+      const existing = store.getFileByPath(relPath);
+
+      // Fast path: mtime+size unchanged → DB row is current
+      if (existing && existing.mtime === mtime && existing.size === size) {
+        skipped++;
+        continue;
+      }
+
+      // Skip large files (extractor cap matches runSyncV2 / applyIncrementalV2)
+      if (size > 1024 * 1024) { skipped++; continue; }
+
+      // Extract — single source of truth, same path runSyncV2 uses
+      const result = extractFile(relPath, projectRoot);
+      if (!result) { skipped++; continue; }
+
+      // Hash-equal but mtime drifted (e.g. `touch foo.ts`) — just refresh
+      // the cached mtime/size so future stat checks short-circuit.
+      if (existing && existing.hash === result.hash) {
+        store.updateFileMtime(relPath, mtime, size);
+        skipped++;
+        continue;
+      }
+
+      const fileId = store.upsertFile(relPath, {
+        language: result.language,
+        hash: result.hash,
+        mtime: result.mtime,
+        size: result.size
+      });
+
+      const resolvedImports = (result.imports || []).map(impPath => {
+        const resolved = store.getFileByPath(impPath);
+        return { path: impPath, resolvedFileId: resolved ? resolved.id : null };
+      });
+
+      const symbols = (result.extracted.functions || []).map(name => ({
+        name: typeof name === 'string' ? name : name.name || 'unknown',
+        kind: 'function',
+        exported: true
+      }));
+
+      store.storeExtraction(fileId, {
+        imports: resolvedImports,
+        symbols,
+        routes: result.extracted.routes || [],
+        models: result.extracted.models || [],
+        envVars: result.extracted.envVars || [],
+        dbTables: (result.extracted.dbTables || []).map(t => ({
+          table: t.table || t.table_name || t.name,
+          operation: t.operation
+        })),
+        errors: result.errors || []
+      });
+
+      // Repair any prior unresolved imports that point AT this file —
+      // e.g. another file imported foo.ts before foo.ts was indexed.
+      // Now foo.ts exists, so those imports become resolvable. Cheap:
+      // single UPDATE WHERE to_path = relPath.
+      store.resolveUnresolvedImports();
+
+      updateReverseDepsPartial(store, fileId);
+      reparsed++;
+    }
+
+    // Track partial-sync timestamp so observability tools can tell apart a
+    // full reindex from a hot reparse.
+    if (reparsed > 0 || removed > 0) {
+      try { store.setMeta('last_partial_sync', new Date().toISOString()); } catch {}
+      // Keep extraction_error_count fresh on partial syncs too
+      try { store.setMeta('extraction_error_count', String(store.getExtractionErrorCount())); } catch {}
+      // Invalidate the bitmap sidecar so the next MCP query
+      // rebuilds it from the now-fresh SQLite state. Removes both the
+      // in-memory singleton and the on-disk `bitmap.bin` so a different
+      // process (e.g. the MCP server when watch.js triggered the
+      // partial sync) also picks up the change.
+      try { invalidateBitmap(path.join(projectRoot, '.carto')); } catch {}
+    }
+  } finally {
+    if (ownsStore) {
+      try { store.close(); } catch {}
+    }
+  }
+
+  return { reparsed, skipped, removed, elapsed: Date.now() - startTime };
+}
+
+/**
+ * updateReverseDepsPartial(store, fileId)
+ *
+ * Recompute reverse_deps for `fileId` plus its direct import neighbors only.
+ * O(neighbors) instead of O(all files). Used by syncFiles() and watch.js.
+ *
+ * Lifted out of the original watch.js implementation so MCP-side lazy
+ * reparse and the watcher share one code path.
+ */
+function updateReverseDepsPartial(store, fileId) {
+  const outgoing = store.db.prepare(
+    'SELECT to_file_id FROM imports WHERE from_file_id = ? AND to_file_id IS NOT NULL'
+  ).all(fileId).map(r => r.to_file_id);
+
+  const incoming = store.db.prepare(
+    'SELECT from_file_id FROM imports WHERE to_file_id = ?'
+  ).all(fileId).map(r => r.from_file_id);
+
+  const affectedIds = new Set([fileId, ...outgoing, ...incoming]);
+
+  const del = store.db.prepare('DELETE FROM reverse_deps WHERE file_id = ? OR dependent_file_id = ?');
+  const ins = store.db.prepare(
+    'INSERT OR IGNORE INTO reverse_deps (file_id, dependent_file_id, hop_distance) VALUES (?,?,?)'
+  );
+
+  // Build local reverse-edge map ONCE, reuse across affected ids.
+  const allEdges = store.db.prepare(
+    'SELECT from_file_id, to_file_id FROM imports WHERE to_file_id IS NOT NULL'
+  ).all();
+
+  const reverseDirect = new Map();
+  for (const e of allEdges) {
+    if (!reverseDirect.has(e.to_file_id)) reverseDirect.set(e.to_file_id, []);
+    reverseDirect.get(e.to_file_id).push(e.from_file_id);
+  }
+
+  const tx = store.db.transaction(() => {
+    for (const fid of affectedIds) {
+      del.run(fid, fid);
+      let frontier = new Set(reverseDirect.get(fid) || []);
+      const visited = new Set();
+      for (let hop = 1; hop <= 5; hop++) {
+        const next = new Set();
+        for (const depId of frontier) {
+          if (visited.has(depId) || depId === fid) continue;
+          visited.add(depId);
+          ins.run(fid, depId, hop);
+          for (const nd of (reverseDirect.get(depId) || [])) {
+            if (!visited.has(nd) && nd !== fid) next.add(nd);
+          }
+        }
+        if (next.size === 0) break;
+        frontier = next;
+      }
+    }
+  });
+  tx();
+
+  const updateCentrality = store.db.prepare(
+    'UPDATE files SET centrality = (SELECT COUNT(DISTINCT dependent_file_id) FROM reverse_deps WHERE file_id = files.id) WHERE id = ?'
+  );
+  for (const fid of affectedIds) updateCentrality.run(fid);
+}
+
+module.exports = { runSyncV2, syncFiles, updateReverseDepsPartial, discoverFiles, extractFile, buildFileDataFromStore, generateOutputs, buildRoutesByFile, detectLanguage, selectClusteringStrategy };