carto-md 2.0.7 → 2.0.8

package/src/bitmap/tools.js

@@ -0,0 +1,310 @@
+'use strict';
+
+/**
+ * Production bitmap query tools — drop-in replacements for 5 SQLiteStore
+ * methods plus one new tool that's only feasible with bitmap aggregation.
+ *
+ * Each function returns the same shape as the corresponding
+ * `SQLiteStore` method so the MCP layer's formatting code (`server-v2.js`)
+ * is data-source-agnostic — bitmap and SQLite paths render identically.
+ *
+ * Performance contract (measured against the SQLite query path):
+ *   blastRadius        20-50× faster on small/medium repos, 5-9× on largest
+ *   crossDomain        4-12×
+ *   highImpactFiles    O(N) → O(1) array slice via popcountIndex
+ *   similarPatterns    20-150× faster (Jaccard over import sets)
+ *   simulateChangeImpact  qualitatively new — N×SQL approach is too slow to
+ *                      ship as a tool; bitmap OR-aggregate is sub-millisecond
+ */
+
+const { Bitset } = require('./bitset');
+
+/**
+ * blastRadius(sidecar, file, maxHops=5) → [{file, hop_distance}] | null
+ *
+ * BFS over reverse adjacency bitmaps. Tracks the hop at which each
+ * dependent first becomes reachable so the output matches `SQLiteStore.
+ * getBlastRadius` row-for-row.
+ *
+ * Returns null if `file` is not in the index — same null contract as
+ * the SQLite version, so the MCP "File not found in index" message
+ * works without a code change.
+ *
+ * Pre-allocates `visited`, `frontier`, and a single transient
+ * `next` bitset; all word-level ops applied in place via `orInPlace` /
+ * `andNotInPlace` / `copyFrom`. Allocation count: 3 Bitsets per call,
+ * regardless of hop depth.
+ */
+function blastRadius(sidecar, file, maxHops = 5) {
+  const fileId = sidecar.pathToFileId.get(file);
+  if (fileId === undefined) return null;
+
+  const { reverse, size, filePathArr } = sidecar;
+
+  const visited = new Bitset(size);
+  visited.set(fileId);
+  const frontier = new Bitset(size);
+  frontier.set(fileId);
+  const next = new Bitset(size);
+
+  // hopOf[id] = first hop at which id became reachable.
+  const hopOf = new Map();
+
+  for (let hop = 1; hop <= maxHops; hop++) {
+    next.setAll(0);
+    // For each bit in `frontier`, OR its reverse-adjacency bitmap into `next`.
+    const fwords = frontier.words;
+    for (let w = 0; w < fwords.length; w++) {
+      let v = fwords[w];
+      while (v) {
+        const bit = v & -v;
+        const fid = (w << 5) + (31 - Math.clz32(bit));
+        v ^= bit;
+        const deps = reverse.get(fid);
+        if (deps) next.orInPlace(deps);
+      }
+    }
+    next.andNotInPlace(visited);
+    if (next.popcount() === 0) break;
+    // Record the first hop at which each new bit became reachable.
+    const nwords = next.words;
+    for (let w = 0; w < nwords.length; w++) {
+      let v = nwords[w];
+      while (v) {
+        const bit = v & -v;
+        const id = (w << 5) + (31 - Math.clz32(bit));
+        v ^= bit;
+        if (!hopOf.has(id)) hopOf.set(id, hop);
+      }
+    }
+    visited.orInPlace(next);
+    frontier.copyFrom(next);
+  }
+
+  // Sort by hop ASC then path ASC — same ORDER BY as SQLite version.
+  // Use byte comparison (`<`/`>`) instead of `localeCompare`: matches
+  // SQLite's BINARY collation default and runs ~10× faster for ASCII paths.
+  const rows = [];
+  for (const [id, hop] of hopOf) {
+    const p = filePathArr[id];
+    if (p !== undefined) rows.push({ file: p, hop_distance: hop });
+  }
+  rows.sort((a, b) => {
+    if (a.hop_distance !== b.hop_distance) return a.hop_distance - b.hop_distance;
+    if (a.file < b.file) return -1;
+    if (a.file > b.file) return 1;
+    return 0;
+  });
+  return rows;
+}
+
+/**
+ * crossDomain(sidecar) → [{from, fromDomain, to, toDomain}]
+ *
+ * Iterates forward edges, emits the row whenever `from` and `to` belong to
+ * different domains. Drop-in for `SQLiteStore.getCrossDomainDeps`.
+ *
+ * Stable sort by (fromDomain, toDomain, from, to) matches the SQL
+ * ORDER BY (BINARY collation) so the output is byte-identical to the
+ * SQLite path on a given DB snapshot for ASCII paths.
+ *
+ * Uses `fileDomainArr` (Int32Array index) and
+ * `filePathArr` / `domainNameArr` (plain Array index) to avoid the
+ * 4× `Map.get` overhead per edge. The hot loop iterates `crossForward` —
+ * pre-masked at sidecar build time so intra-domain edges (95%+ of
+ * imports on real repos) are already excluded; no per-edge same-domain
+ * check, no wasted bit iteration.
+ */
+function crossDomain(sidecar) {
+  const { crossForward, fileDomainArr, filePathArr, domainNameArr } = sidecar;
+  const rows = [];
+  for (const [fromId, bitmap] of crossForward) {
+    const fromDomainId = fileDomainArr[fromId];
+    if (fromDomainId < 0) continue;
+    const fromDomain = domainNameArr[fromDomainId];
+    if (!fromDomain) continue;
+    const fromPath = filePathArr[fromId];
+    if (fromPath === undefined) continue;
+    const words = bitmap.words;
+    for (let w = 0; w < words.length; w++) {
+      let v = words[w];
+      while (v) {
+        const bit = v & -v;
+        const toId = (w << 5) + (31 - Math.clz32(bit));
+        v ^= bit;
+        const toDomainId = fileDomainArr[toId];
+        // crossForward already masked out same-domain bits, but a file
+        // with no domain assignment can still be present — skip those
+        // (they have no toDomain name to emit).
+        if (toDomainId < 0) continue;
+        const toDomain = domainNameArr[toDomainId];
+        if (!toDomain) continue;
+        const toPath = filePathArr[toId];
+        if (toPath === undefined) continue;
+        rows.push({ from: fromPath, fromDomain, to: toPath, toDomain });
+      }
+    }
+  }
+  rows.sort((a, b) => {
+    if (a.fromDomain < b.fromDomain) return -1;
+    if (a.fromDomain > b.fromDomain) return 1;
+    if (a.toDomain < b.toDomain) return -1;
+    if (a.toDomain > b.toDomain) return 1;
+    if (a.from < b.from) return -1;
+    if (a.from > b.from) return 1;
+    if (a.to < b.to) return -1;
+    if (a.to > b.to) return 1;
+    return 0;
+  });
+  return rows;
+}
+
+/**
+ * highImpactFiles(sidecar, limit=10) → [{file, dependents}]
+ *
+ * O(1) array slice over `popcountIndex` — the sidecar maintains a list
+ * of file ids sorted DESC by transitive dependent count at build time,
+ * so the query at runtime is just `popcountIndex.slice(0, limit)`. No
+ * popcount, no sort, no allocation.
+ *
+ * Output shape mirrors `SQLiteStore.getHighImpactFiles` exactly.
+ */
+function highImpactFiles(sidecar, limit = 10) {
+  const out = [];
+  const idx = sidecar.popcountIndex;
+  const n = Math.min(limit, idx.length);
+  for (let i = 0; i < n; i++) {
+    const p = sidecar.fileIdToPath.get(idx[i].fileId);
+    if (p) out.push({ file: p, dependents: idx[i].count });
+  }
+  return out;
+}
+
+/**
+ * similarPatterns(sidecar, file, k=5) → [{file, score, shared}]
+ *
+ * Jaccard similarity over forward-import sets — for each candidate file,
+ * `score = |A ∩ B| / |A ∪ B|`. Returns the top-K most similar files.
+ *
+ * This semantics differs from the legacy SQLite tool (which used three
+ * separate SQL strategies — same domain, same route methods, shared
+ * imports). Both answer "what does this file look like elsewhere?", but
+ * Jaccard is the standard graph-similarity metric and runs in
+ * microseconds where the 3-strategy SQL took milliseconds. The output
+ * is structured so server-v2 can format a simple, focused result block.
+ *
+ * Returns `[]` when the target has no resolved imports — honest signal
+ * that there's nothing structural to compare against.
+ */
+function similarPatterns(sidecar, file, k = 5) {
+  const fileId = sidecar.pathToFileId.get(file);
+  if (fileId === undefined) return null;
+  const target = sidecar.forward.get(fileId);
+  if (!target || target.popcount() === 0) return [];
+
+  const scores = [];
+  for (const [otherId, bitmap] of sidecar.forward) {
+    if (otherId === fileId) continue;
+    const intersection = target.and(bitmap).popcount();
+    if (intersection === 0) continue;
+    const union = target.or(bitmap).popcount();
+    const p = sidecar.fileIdToPath.get(otherId);
+    if (!p) continue;
+    scores.push({ file: p, score: intersection / union, shared: intersection });
+  }
+  scores.sort((a, b) => b.score - a.score || b.shared - a.shared);
+  return scores.slice(0, k);
+}
+
+/**
+ * simulateChangeImpact(sidecar, files) → { files: [{file, hop_distance}], count }
+ *
+ * **New tool — only feasible with bitmaps.** Given a *set* of files
+ * changing simultaneously, returns the union of every transitively
+ * affected file. Equivalent to N parallel reverse-BFS calls aggregated
+ * via bitwise OR — O(F + E) bitmap ops vs O(N×F×E) SQL queries.
+ *
+ * Hop distance is the *minimum* hop at which a dependent becomes
+ * reachable from any of the input files (BFS frontier OR-aggregated
+ * across all sources at each hop).
+ *
+ * Input files that aren't in the index are ignored silently. The
+ * returned `files` array excludes the input set itself (you don't want
+ * to count "this file depends on this file" as impact).
+ *
+ * Accepts paths (strings) or pre-resolved file ids (numbers) for
+ * convenience — the MCP tool dispatcher sends paths; tests use ids.
+ */
+function simulateChangeImpact(sidecar, files, maxHops = 5) {
+  const { reverse, size, filePathArr, pathToFileId } = sidecar;
+
+  const seedIds = [];
+  for (const f of files || []) {
+    if (typeof f === 'number') {
+      if (f >= 0 && f < size && filePathArr[f] !== undefined) seedIds.push(f);
+    } else if (typeof f === 'string') {
+      const id = pathToFileId.get(f);
+      if (id !== undefined) seedIds.push(id);
+    }
+  }
+  if (seedIds.length === 0) return { files: [], count: 0 };
+
+  const visited = new Bitset(size);
+  const frontier = new Bitset(size);
+  for (const fid of seedIds) {
+    visited.set(fid);
+    frontier.set(fid);
+  }
+  const next = new Bitset(size);
+
+  const hopOf = new Map();
+  for (let hop = 1; hop <= maxHops; hop++) {
+    next.setAll(0);
+    const fwords = frontier.words;
+    for (let w = 0; w < fwords.length; w++) {
+      let v = fwords[w];
+      while (v) {
+        const bit = v & -v;
+        const fid = (w << 5) + (31 - Math.clz32(bit));
+        v ^= bit;
+        const deps = reverse.get(fid);
+        if (deps) next.orInPlace(deps);
+      }
+    }
+    next.andNotInPlace(visited);
+    if (next.popcount() === 0) break;
+    const nwords = next.words;
+    for (let w = 0; w < nwords.length; w++) {
+      let v = nwords[w];
+      while (v) {
+        const bit = v & -v;
+        const id = (w << 5) + (31 - Math.clz32(bit));
+        v ^= bit;
+        if (!hopOf.has(id)) hopOf.set(id, hop);
+      }
+    }
+    visited.orInPlace(next);
+    frontier.copyFrom(next);
+  }
+
+  const rows = [];
+  for (const [id, hop] of hopOf) {
+    const p = filePathArr[id];
+    if (p !== undefined) rows.push({ file: p, hop_distance: hop });
+  }
+  rows.sort((a, b) => {
+    if (a.hop_distance !== b.hop_distance) return a.hop_distance - b.hop_distance;
+    if (a.file < b.file) return -1;
+    if (a.file > b.file) return 1;
+    return 0;
+  });
+  return { files: rows, count: rows.length };
+}
+
+module.exports = {
+  blastRadius,
+  crossDomain,
+  highImpactFiles,
+  similarPatterns,
+  simulateChangeImpact,
+};

package/src/cli/check.js

@@ -39,6 +39,14 @@ async function run(projectRoot) {
     const ageStr = age < 60 ? `${age}s ago` : age < 3600 ? `${Math.round(age / 60)}m ago` : `${Math.round(age / 3600)}h ago`;
     console.log(`  Last indexed  : ${ageStr}`);
   }
+
+  // Language coverage (only shows when grammars are missing)
+  const unavailRaw = store.getMeta('unavailable_languages_json');
+  const unavailLangs = unavailRaw ? (() => { try { return JSON.parse(unavailRaw); } catch { return []; } })() : [];
+  if (unavailLangs.length > 0) {
+    console.log(`  Lang coverage : ⚠️  ${unavailLangs.length} grammar${unavailLangs.length === 1 ? '' : 's'} unavailable (${unavailLangs.join(', ')}) — regex fallback active`);
+  }
+
   console.log('');
 
   // ── Uncommitted changes that touch high-blast-radius files ───────────────
@@ -89,6 +97,55 @@ async function run(projectRoot) {
     console.log('  ✅ No cross-domain dependency violations\n');
   }
 
+  // ── Domain stability ──────────────────────────────────────────────────
+  const driftPct = parseFloat(store.getMeta('domain_stability_drift_pct') || '0');
+  const reassignmentsRaw = store.getMeta('last_reassignments_json');
+  const reassignments = reassignmentsRaw ? (() => { try { return JSON.parse(reassignmentsRaw); } catch { return []; } })() : [];
+
+  if (driftPct > 0 || reassignments.length > 0) {
+    if (driftPct > 5) {
+      hasIssues = true;
+      console.log(`  ⚠️  Domain stability: ${driftPct.toFixed(1)}% drift`);
+    } else {
+      console.log(`  ✅ Domain stability: ${driftPct.toFixed(1)}% drift`);
+    }
+    if (reassignments.length > 0) {
+      console.log('     Recent reassignments:');
+      for (const r of reassignments.slice(0, 5)) {
+        console.log(`       ${path.basename(r.file)}: ${r.from} → ${r.to}`);
+      }
+      if (reassignments.length > 5) console.log(`       ...and ${reassignments.length - 5} more`);
+    }
+    console.log('');
+  } else if (domains.length > 1) {
+    console.log('  ✅ Domain stability: 0.0% drift (no reassignments)\n');
+  }
+
+  // ── Extraction errors ─────────────────────────────────────────────────
+  const errorCount = store.getExtractionErrorCount();
+  if (errorCount > 0) {
+    hasIssues = true;
+    const topFiles = store.getExtractionErrorsTopFiles(5);
+    console.log(`  ⚠️  Extraction errors (${errorCount}):`);
+    console.log('     These files failed to parse — their routes/models/imports are missing from the index.\n');
+    for (const f of topFiles) {
+      const phases = f.phases ? ` [${f.phases}]` : '';
+      console.log(`     ${f.file}${phases}`);
+      if (f.sample) {
+        // Truncate sample for terminal readability
+        const sample = String(f.sample).split('\n')[0].slice(0, 120);
+        console.log(`       └─ ${sample}`);
+      }
+    }
+    // Errors NOT covered by the top-5 file slice (i.e., 6th+ file's errors).
+    const shown = topFiles.reduce((a, f) => a + (f.errorCount || 0), 0);
+    const remainingErrors = errorCount - shown;
+    if (remainingErrors > 0) {
+      console.log(`     ...and ${remainingErrors} more error${remainingErrors === 1 ? '' : 's'} in additional files`);
+    }
+    console.log('');
+  }
+
   // ── Top high-impact files ────────────────────────────────────────────────
   if (highImpact.length > 0) {
     console.log(`  🔥 Top high-impact files (changing these = highest blast radius):`);

package/src/cli/index.js

@@ -8,11 +8,18 @@ function printUsage() {
 Usage: carto <command>
 
 Commands:
-  init          Detect project, write .carto/config.json, run first sync
-  watch         Read .carto/config.json, start file watcher
+  init          Detect project, write .carto/config.json, run first sync.
+                Installs git hooks (pre-commit, post-checkout, post-merge,
+                post-rewrite) so the index stays fresh on every git event.
   sync          Read .carto/config.json, run one sync, exit
+  watch         (Optional) Start file watcher for sub-second freshness.
+                Not required — git hooks + lazy MCP re-parse keep the
+                index fresh by default. Use only for AI-heavy workflows.
   impact <file> Show which files and routes are affected by changing a file
   check         Report cross-domain deps, high-risk uncommitted changes, domain health
+  inspect       Read-only diagnostic: prints index paths, sizes, freshness,
+                bitmap sidecar shape, top-impact files, schema version,
+                sync timestamps, extraction errors. Use --json for piping.
   remove        Remove AGENTS.md and .carto/ from this project
   serve         Start MCP server for AI tool integration
   agent         Start ACP agent mode (for Zed, JetBrains, VS Code)
@@ -55,6 +62,11 @@ if (command === 'init') {
     console.error(`[CARTO] Fatal error: ${err.message}`);
     process.exit(1);
   });
+} else if (command === 'inspect') {
+  // Read-only diagnostic — no async, no rebuild. Pass --json through.
+  const json = process.argv.slice(3).includes('--json');
+  const code = require('./inspect').run(process.cwd(), { json });
+  process.exit(code);
 } else if (command === 'remove') {
   require('./remove').run(process.cwd());
 } else if (command === 'serve') {