@optave/codegraph 1.1.0

package/src/embedder.js

@@ -0,0 +1,330 @@
+import fs from "fs";
+import path from "path";
+
+import Database from 'better-sqlite3';
+import { findDbPath, openReadonlyOrFail } from './db.js';
+import { warn, debug } from './logger.js';
+
+// Lazy-load transformers (heavy, optional module)
+let pipeline = null;
+let cos_sim = null;
+let extractor = null;
+let activeModel = null;
+
+export const MODELS = {
+  'minilm': {
+    name: 'Xenova/all-MiniLM-L6-v2',
+    dim: 384,
+    desc: 'Smallest, fastest (~23MB). General text.',
+    quantized: true
+  },
+  'jina-small': {
+    name: 'Xenova/jina-embeddings-v2-small-en',
+    dim: 512,
+    desc: 'Small, good quality (~33MB). General text.',
+    quantized: false
+  },
+  'jina-base': {
+    name: 'Xenova/jina-embeddings-v2-base-en',
+    dim: 768,
+    desc: 'Good quality (~137MB). General text, 8192 token context.',
+    quantized: false
+  },
+  'nomic': {
+    name: 'Xenova/nomic-embed-text-v1',
+    dim: 768,
+    desc: 'Best local quality (~137MB). 8192 context, beats OpenAI ada-002.',
+    quantized: false
+  }
+};
+
+export const DEFAULT_MODEL = 'minilm';
+const BATCH_SIZE_MAP = { 'minilm': 32, 'jina-small': 16, 'jina-base': 8, 'nomic': 8 };
+const DEFAULT_BATCH_SIZE = 32;
+
+function getModelConfig(modelKey) {
+  const key = modelKey || DEFAULT_MODEL;
+  const config = MODELS[key];
+  if (!config) {
+    console.error(`Unknown model: ${key}. Available: ${Object.keys(MODELS).join(', ')}`);
+    process.exit(1);
+  }
+  return config;
+}
+
+/**
+ * Lazy-load @huggingface/transformers.
+ * This is an optional dependency — gives a clear error if not installed.
+ */
+async function loadTransformers() {
+  try {
+    return await import('@huggingface/transformers');
+  } catch {
+    console.error(
+      'Semantic search requires @huggingface/transformers.\n' +
+      'Install it with: npm install @huggingface/transformers'
+    );
+    process.exit(1);
+  }
+}
+
+async function loadModel(modelKey) {
+  const config = getModelConfig(modelKey);
+
+  if (extractor && activeModel === config.name) return { extractor, config };
+
+  const transformers = await loadTransformers();
+  pipeline = transformers.pipeline;
+  cos_sim = transformers.cos_sim;
+
+  console.log(`Loading embedding model: ${config.name} (${config.dim}d)...`);
+  const opts = config.quantized ? { quantized: true } : {};
+  extractor = await pipeline('feature-extraction', config.name, opts);
+  activeModel = config.name;
+  console.log('Model loaded.');
+  return { extractor, config };
+}
+
+/**
+ * Generate embeddings for an array of texts.
+ */
+export async function embed(texts, modelKey) {
+  const { extractor: ext, config } = await loadModel(modelKey);
+  const dim = config.dim;
+  const results = [];
+  const batchSize = BATCH_SIZE_MAP[modelKey || DEFAULT_MODEL] || DEFAULT_BATCH_SIZE;
+
+  for (let i = 0; i < texts.length; i += batchSize) {
+    const batch = texts.slice(i, i + batchSize);
+    const output = await ext(batch, { pooling: 'mean', normalize: true });
+
+    for (let j = 0; j < batch.length; j++) {
+      const start = j * dim;
+      const vec = new Float32Array(dim);
+      for (let k = 0; k < dim; k++) {
+        vec[k] = output.data[start + k];
+      }
+      results.push(vec);
+    }
+
+    if (texts.length > batchSize) {
+      process.stdout.write(`  Embedded ${Math.min(i + batchSize, texts.length)}/${texts.length}\r`);
+    }
+  }
+
+  return { vectors: results, dim };
+}
+
+/**
+ * Cosine similarity between two Float32Arrays.
+ */
+export function cosineSim(a, b) {
+  let dot = 0, normA = 0, normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+  return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+
+function initEmbeddingsSchema(db) {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS embeddings (
+      node_id INTEGER PRIMARY KEY,
+      vector BLOB NOT NULL,
+      text_preview TEXT,
+      FOREIGN KEY(node_id) REFERENCES nodes(id)
+    );
+    CREATE TABLE IF NOT EXISTS embedding_meta (
+      key TEXT PRIMARY KEY,
+      value TEXT
+    );
+  `);
+}
+
+/**
+ * Build embeddings for all functions/methods/classes in the graph.
+ */
+export async function buildEmbeddings(rootDir, modelKey) {
+  // path already imported at top
+  // fs already imported at top
+  const dbPath = findDbPath(null);
+
+  const db = new Database(dbPath);
+  initEmbeddingsSchema(db);
+
+  db.exec('DELETE FROM embeddings');
+  db.exec('DELETE FROM embedding_meta');
+
+  const nodes = db.prepare(
+    `SELECT * FROM nodes WHERE kind IN ('function', 'method', 'class') ORDER BY file, line`
+  ).all();
+
+  console.log(`Building embeddings for ${nodes.length} symbols...`);
+
+  const byFile = new Map();
+  for (const node of nodes) {
+    if (!byFile.has(node.file)) byFile.set(node.file, []);
+    byFile.get(node.file).push(node);
+  }
+
+  const texts = [];
+  const nodeIds = [];
+  const previews = [];
+
+  for (const [file, fileNodes] of byFile) {
+    const fullPath = path.join(rootDir, file);
+    let lines;
+    try {
+      lines = fs.readFileSync(fullPath, 'utf-8').split('\n');
+    } catch (err) {
+      warn(`Cannot read ${file} for embeddings: ${err.message}`);
+      continue;
+    }
+
+    for (const node of fileNodes) {
+      const startLine = Math.max(0, node.line - 1);
+      const endLine = node.end_line
+        ? Math.min(lines.length, node.end_line)
+        : Math.min(lines.length, startLine + 15);
+      const context = lines.slice(startLine, endLine).join('\n');
+
+      const text = `${node.kind} ${node.name} in ${file}\n${context}`;
+      texts.push(text);
+      nodeIds.push(node.id);
+      previews.push(`${node.name} (${node.kind}) -- ${file}:${node.line}`);
+    }
+  }
+
+  console.log(`Embedding ${texts.length} symbols...`);
+  const { vectors, dim } = await embed(texts, modelKey);
+
+  const insert = db.prepare('INSERT OR REPLACE INTO embeddings (node_id, vector, text_preview) VALUES (?, ?, ?)');
+  const insertMeta = db.prepare('INSERT OR REPLACE INTO embedding_meta (key, value) VALUES (?, ?)');
+  const insertAll = db.transaction(() => {
+    for (let i = 0; i < vectors.length; i++) {
+      insert.run(nodeIds[i], Buffer.from(vectors[i].buffer), previews[i]);
+    }
+    const config = getModelConfig(modelKey);
+    insertMeta.run('model', config.name);
+    insertMeta.run('dim', String(dim));
+    insertMeta.run('count', String(vectors.length));
+    insertMeta.run('built_at', new Date().toISOString());
+  });
+  insertAll();
+
+  console.log(`\nStored ${vectors.length} embeddings (${dim}d, ${getModelConfig(modelKey).name}) in graph.db`);
+  db.close();
+}
+
+/**
+ * Semantic search with pre-filter support to reduce the search space.
+ */
+export async function search(query, customDbPath, opts = {}) {
+  const limit = opts.limit || 15;
+  const noTests = opts.noTests || false;
+  const minScore = opts.minScore || 0.2;
+
+  const db = openReadonlyOrFail(customDbPath);
+
+  let count;
+  try {
+    count = db.prepare("SELECT COUNT(*) as c FROM embeddings").get().c;
+  } catch {
+    console.log('No embeddings table found. Run `codegraph embed` first.');
+    db.close();
+    return;
+  }
+  if (count === 0) {
+    console.log('No embeddings found. Run `codegraph embed` first.');
+    db.close();
+    return;
+  }
+
+  let storedModel = null;
+  let storedDim = null;
+  try {
+    const modelRow = db.prepare("SELECT value FROM embedding_meta WHERE key = 'model'").get();
+    const dimRow = db.prepare("SELECT value FROM embedding_meta WHERE key = 'dim'").get();
+    if (modelRow) storedModel = modelRow.value;
+    if (dimRow) storedDim = parseInt(dimRow.value);
+  } catch { /* old DB without meta table */ }
+
+  let modelKey = opts.model || null;
+  if (!modelKey && storedModel) {
+    for (const [key, config] of Object.entries(MODELS)) {
+      if (config.name === storedModel) { modelKey = key; break; }
+    }
+  }
+
+  const { vectors: [queryVec], dim } = await embed([query], modelKey);
+
+  if (storedDim && dim !== storedDim) {
+    console.log(`Warning: query model dimension (${dim}) doesn't match stored embeddings (${storedDim}).`);
+    console.log(`  Re-run \`codegraph embed\` with the same model, or use --model to match.`);
+    db.close();
+    return;
+  }
+
+  // Pre-filter: allow filtering by kind or file pattern to reduce search space
+  const TEST_PATTERN = /\.(test|spec)\.|__test__|__tests__|\.stories\./;
+  let sql = `
+    SELECT e.node_id, e.vector, e.text_preview, n.name, n.kind, n.file, n.line
+    FROM embeddings e
+    JOIN nodes n ON e.node_id = n.id
+  `;
+  const params = [];
+  const conditions = [];
+  if (opts.kind) {
+    conditions.push('n.kind = ?');
+    params.push(opts.kind);
+  }
+  if (opts.filePattern) {
+    conditions.push('n.file LIKE ?');
+    params.push(`%${opts.filePattern}%`);
+  }
+  if (conditions.length > 0) {
+    sql += ' WHERE ' + conditions.join(' AND ');
+  }
+
+  const rows = db.prepare(sql).all(...params);
+
+  const results = [];
+  for (const row of rows) {
+    if (noTests && TEST_PATTERN.test(row.file)) continue;
+
+    const vec = new Float32Array(new Uint8Array(row.vector).buffer);
+    const sim = cosineSim(queryVec, vec);
+
+    if (sim >= minScore) {
+      results.push({
+        name: row.name,
+        kind: row.kind,
+        file: row.file,
+        line: row.line,
+        similarity: sim
+      });
+    }
+  }
+
+  results.sort((a, b) => b.similarity - a.similarity);
+
+  console.log(`\nSemantic search: "${query}"\n`);
+
+  const topResults = results.slice(0, limit);
+  if (topResults.length === 0) {
+    console.log('  No results above threshold.');
+  } else {
+    for (const r of topResults) {
+      const bar = '#'.repeat(Math.round(r.similarity * 20));
+      const kindIcon = r.kind === 'function' ? 'f' : r.kind === 'class' ? '*' : 'o';
+      console.log(`  ${(r.similarity * 100).toFixed(1)}% ${bar}`);
+      console.log(`    ${kindIcon} ${r.name} -- ${r.file}:${r.line}`);
+    }
+  }
+
+  console.log(`\n  ${results.length} results total (showing top ${topResults.length})\n`);
+  db.close();
+}
+

package/src/export.js

@@ -0,0 +1,138 @@
+import path from 'path';
+
+/**
+ * Export the dependency graph in DOT (Graphviz) format.
+ */
+export function exportDOT(db, opts = {}) {
+  const fileLevel = opts.fileLevel !== false;
+  const lines = [
+    'digraph codegraph {',
+    '  rankdir=LR;',
+    '  node [shape=box, fontname="monospace", fontsize=10];',
+    '  edge [color="#666666"];',
+    ''
+  ];
+
+  if (fileLevel) {
+    const edges = db.prepare(`
+      SELECT DISTINCT n1.file AS source, n2.file AS target
+      FROM edges e
+      JOIN nodes n1 ON e.source_id = n1.id
+      JOIN nodes n2 ON e.target_id = n2.id
+      WHERE n1.file != n2.file AND e.kind IN ('imports', 'imports-type', 'calls')
+    `).all();
+
+    const dirs = new Map();
+    const allFiles = new Set();
+    for (const { source, target } of edges) {
+      allFiles.add(source);
+      allFiles.add(target);
+    }
+    for (const file of allFiles) {
+      const dir = path.dirname(file) || '.';
+      if (!dirs.has(dir)) dirs.set(dir, []);
+      dirs.get(dir).push(file);
+    }
+
+    let clusterIdx = 0;
+    for (const [dir, files] of [...dirs].sort()) {
+      lines.push(`  subgraph cluster_${clusterIdx++} {`);
+      lines.push(`    label="${dir}";`);
+      lines.push(`    style=dashed;`);
+      lines.push(`    color="#999999";`);
+      for (const f of files) {
+        const label = path.basename(f);
+        lines.push(`    "${f}" [label="${label}"];`);
+      }
+      lines.push(`  }`);
+      lines.push('');
+    }
+
+    for (const { source, target } of edges) {
+      lines.push(`  "${source}" -> "${target}";`);
+    }
+  } else {
+    const edges = db.prepare(`
+      SELECT n1.name AS source_name, n1.kind AS source_kind, n1.file AS source_file,
+             n2.name AS target_name, n2.kind AS target_kind, n2.file AS target_file,
+             e.kind AS edge_kind
+      FROM edges e
+      JOIN nodes n1 ON e.source_id = n1.id
+      JOIN nodes n2 ON e.target_id = n2.id
+      WHERE n1.kind IN ('function', 'method', 'class') AND n2.kind IN ('function', 'method', 'class')
+      AND e.kind = 'calls'
+    `).all();
+
+    for (const e of edges) {
+      const sId = `${e.source_file}:${e.source_name}`.replace(/[^a-zA-Z0-9_]/g, '_');
+      const tId = `${e.target_file}:${e.target_name}`.replace(/[^a-zA-Z0-9_]/g, '_');
+      lines.push(`  ${sId} [label="${e.source_name}\\n${path.basename(e.source_file)}"];`);
+      lines.push(`  ${tId} [label="${e.target_name}\\n${path.basename(e.target_file)}"];`);
+      lines.push(`  ${sId} -> ${tId};`);
+    }
+  }
+
+  lines.push('}');
+  return lines.join('\n');
+}
+
+/**
+ * Export the dependency graph in Mermaid format.
+ */
+export function exportMermaid(db, opts = {}) {
+  const fileLevel = opts.fileLevel !== false;
+  const lines = ['graph LR'];
+
+  if (fileLevel) {
+    const edges = db.prepare(`
+      SELECT DISTINCT n1.file AS source, n2.file AS target
+      FROM edges e
+      JOIN nodes n1 ON e.source_id = n1.id
+      JOIN nodes n2 ON e.target_id = n2.id
+      WHERE n1.file != n2.file AND e.kind IN ('imports', 'imports-type', 'calls')
+    `).all();
+
+    for (const { source, target } of edges) {
+      const s = source.replace(/[^a-zA-Z0-9]/g, '_');
+      const t = target.replace(/[^a-zA-Z0-9]/g, '_');
+      lines.push(`  ${s}["${source}"] --> ${t}["${target}"]`);
+    }
+  } else {
+    const edges = db.prepare(`
+      SELECT n1.name AS source_name, n1.file AS source_file,
+             n2.name AS target_name, n2.file AS target_file
+      FROM edges e
+      JOIN nodes n1 ON e.source_id = n1.id
+      JOIN nodes n2 ON e.target_id = n2.id
+      WHERE n1.kind IN ('function', 'method', 'class') AND n2.kind IN ('function', 'method', 'class')
+      AND e.kind = 'calls'
+    `).all();
+
+    for (const e of edges) {
+      const sId = `${e.source_file}_${e.source_name}`.replace(/[^a-zA-Z0-9]/g, '_');
+      const tId = `${e.target_file}_${e.target_name}`.replace(/[^a-zA-Z0-9]/g, '_');
+      lines.push(`  ${sId}["${e.source_name}"] --> ${tId}["${e.target_name}"]`);
+    }
+  }
+
+  return lines.join('\n');
+}
+
+/**
+ * Export as JSON adjacency list.
+ */
+export function exportJSON(db) {
+  const nodes = db.prepare(`
+    SELECT id, name, kind, file, line FROM nodes WHERE kind = 'file'
+  `).all();
+
+  const edges = db.prepare(`
+    SELECT DISTINCT n1.file AS source, n2.file AS target, e.kind
+    FROM edges e
+    JOIN nodes n1 ON e.source_id = n1.id
+    JOIN nodes n2 ON e.target_id = n2.id
+    WHERE n1.file != n2.file
+  `).all();
+
+  return { nodes, edges };
+}

package/src/index.js

@@ -0,0 +1,39 @@
+/**
+ * codegraph — Programmatic API
+ *
+ * Usage:
+ *   import { buildGraph, queryNameData, findCycles, exportDOT } from 'codegraph';
+ */
+
+// Graph building
+export { buildGraph, resolveImportPath, collectFiles, loadPathAliases } from './builder.js';
+
+// Query functions (data-returning)
+export {
+  queryNameData, impactAnalysisData, moduleMapData,
+  fileDepsData, fnDepsData, fnImpactData, diffImpactData
+} from './queries.js';
+
+// Watch mode
+export { watchProject } from './watcher.js';
+
+// Export (DOT/Mermaid/JSON)
+export { exportDOT, exportMermaid, exportJSON } from './export.js';
+
+// Circular dependency detection
+export { findCycles, formatCycles } from './cycles.js';
+
+// Embeddings
+export { buildEmbeddings, search, embed, cosineSim, MODELS, DEFAULT_MODEL } from './embedder.js';
+
+// Database utilities
+export { openDb, initSchema, findDbPath, openReadonlyOrFail } from './db.js';
+
+// Configuration
+export { loadConfig } from './config.js';
+
+// Shared constants
+export { EXTENSIONS, IGNORE_DIRS, normalizePath } from './constants.js';
+
+// Logger
+export { setVerbose } from './logger.js';

package/src/logger.js

@@ -0,0 +1,20 @@
+let verbose = false;
+
+export function setVerbose(v) { verbose = v; }
+export function isVerbose() { return verbose; }
+
+export function warn(msg) {
+  process.stderr.write(`[codegraph WARN] ${msg}\n`);
+}
+
+export function debug(msg) {
+  if (verbose) process.stderr.write(`[codegraph DEBUG] ${msg}\n`);
+}
+
+export function info(msg) {
+  process.stderr.write(`[codegraph] ${msg}\n`);
+}
+
+export function error(msg) {
+  process.stderr.write(`[codegraph ERROR] ${msg}\n`);
+}

package/src/mcp.js

@@ -0,0 +1,139 @@
+/**
+ * MCP (Model Context Protocol) server for codegraph.
+ * Exposes codegraph queries as tools that AI coding assistants can call.
+ *
+ * Requires: npm install @modelcontextprotocol/sdk
+ */
+
+import { createRequire } from 'node:module';
+import { findDbPath } from './db.js';
+import { findCycles } from './cycles.js';
+
+const TOOLS = [
+  {
+    name: 'query_function',
+    description: 'Find callers and callees of a function by name',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        name: { type: 'string', description: 'Function name to query (supports partial match)' },
+        depth: { type: 'number', description: 'Traversal depth for transitive callers', default: 2 }
+      },
+      required: ['name']
+    }
+  },
+  {
+    name: 'file_deps',
+    description: 'Show what a file imports and what imports it',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        file: { type: 'string', description: 'File path (partial match supported)' }
+      },
+      required: ['file']
+    }
+  },
+  {
+    name: 'impact_analysis',
+    description: 'Show files affected by changes to a given file (transitive)',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        file: { type: 'string', description: 'File path to analyze' }
+      },
+      required: ['file']
+    }
+  },
+  {
+    name: 'find_cycles',
+    description: 'Detect circular dependencies in the codebase',
+    inputSchema: {
+      type: 'object',
+      properties: {}
+    }
+  },
+  {
+    name: 'module_map',
+    description: 'Get high-level overview of most-connected files',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        limit: { type: 'number', description: 'Number of top files to show', default: 20 }
+      }
+    }
+  }
+];
+
+export { TOOLS };
+
+/**
+ * Start the MCP server.
+ * This function requires @modelcontextprotocol/sdk to be installed.
+ */
+export async function startMCPServer(customDbPath) {
+  let Server, StdioServerTransport;
+  try {
+    const sdk = await import('@modelcontextprotocol/sdk/server/index.js');
+    Server = sdk.Server;
+    const transport = await import('@modelcontextprotocol/sdk/server/stdio.js');
+    StdioServerTransport = transport.StdioServerTransport;
+  } catch {
+    console.error(
+      'MCP server requires @modelcontextprotocol/sdk.\n' +
+      'Install it with: npm install @modelcontextprotocol/sdk'
+    );
+    process.exit(1);
+  }
+
+  // Lazy import query functions to avoid circular deps at module load
+  const { queryNameData, impactAnalysisData, moduleMapData, fileDepsData } = await import('./queries.js');
+
+  const require = createRequire(import.meta.url);
+  const Database = require('better-sqlite3');
+
+  const server = new Server(
+    { name: 'codegraph', version: '1.0.0' },
+    { capabilities: { tools: {} } }
+  );
+
+  server.setRequestHandler('tools/list', async () => ({ tools: TOOLS }));
+
+  server.setRequestHandler('tools/call', async (request) => {
+    const { name, arguments: args } = request.params;
+    const dbPath = customDbPath || undefined;
+
+    try {
+      let result;
+      switch (name) {
+        case 'query_function':
+          result = queryNameData(args.name, dbPath);
+          break;
+        case 'file_deps':
+          result = fileDepsData(args.file, dbPath);
+          break;
+        case 'impact_analysis':
+          result = impactAnalysisData(args.file, dbPath);
+          break;
+        case 'find_cycles': {
+          const db = new Database(findDbPath(dbPath), { readonly: true });
+          const cycles = findCycles(db);
+          db.close();
+          result = { cycles, count: cycles.length };
+          break;
+        }
+        case 'module_map':
+          result = moduleMapData(dbPath, args.limit || 20);
+          break;
+        default:
+          return { content: [{ type: 'text', text: `Unknown tool: ${name}` }], isError: true };
+      }
+
+      return { content: [{ type: 'text', text: JSON.stringify(result, null, 2) }] };
+    } catch (err) {
+      return { content: [{ type: 'text', text: `Error: ${err.message}` }], isError: true };
+    }
+  });
+
+  const transport = new StdioServerTransport();
+  await server.connect(transport);
+}