@optave/codegraph 3.1.3 → 3.1.5

package/src/{watcher.js → domain/graph/watcher.js}

@@ -1,13 +1,13 @@
 import fs from 'node:fs';
 import path from 'node:path';
-import { readFileSafe } from './builder.js';
+import { closeDb, getNodeId as getNodeIdQuery, initSchema, openDb } from '../../db/index.js';
+import { info } from '../../infrastructure/logger.js';
+import { EXTENSIONS, IGNORE_DIRS, normalizePath } from '../../shared/constants.js';
+import { DbError } from '../../shared/errors.js';
+import { createParseTreeCache, getActiveEngine } from '../parser.js';
+import { rebuildFile } from './builder/incremental.js';
 import { appendChangeEvents, buildChangeEvent, diffSymbols } from './change-journal.js';
-import { EXTENSIONS, IGNORE_DIRS, normalizePath } from './constants.js';
-import { closeDb, getNodeId as getNodeIdQuery, initSchema, openDb } from './db.js';
 import { appendJournalEntries } from './journal.js';
-import { info, warn } from './logger.js';
-import { createParseTreeCache, getActiveEngine, parseFileIncremental } from './parser.js';
-import { resolveImportPath } from './resolve.js';
 
 function shouldIgnore(filePath) {
   const parts = filePath.split(path.sep);
@@ -18,152 +18,10 @@ function isTrackedExt(filePath) {
   return EXTENSIONS.has(path.extname(filePath));
 }
 
-/**
- * Parse a single file and update the database incrementally.
- */
-async function updateFile(_db, rootDir, filePath, stmts, engineOpts, cache) {
-  const relPath = normalizePath(path.relative(rootDir, filePath));
-
-  const oldNodes = stmts.countNodes.get(relPath)?.c || 0;
-  const _oldEdges = stmts.countEdgesForFile.get(relPath)?.c || 0;
-  const oldSymbols = stmts.listSymbols.all(relPath);
-
-  stmts.deleteEdgesForFile.run(relPath);
-  stmts.deleteNodes.run(relPath);
-
-  if (!fs.existsSync(filePath)) {
-    if (cache) cache.remove(filePath);
-    const symbolDiff = diffSymbols(oldSymbols, []);
-    return {
-      file: relPath,
-      nodesAdded: 0,
-      nodesRemoved: oldNodes,
-      edgesAdded: 0,
-      deleted: true,
-      event: 'deleted',
-      symbolDiff,
-      nodesBefore: oldNodes,
-      nodesAfter: 0,
-    };
-  }
-
-  let code;
-  try {
-    code = readFileSafe(filePath);
-  } catch (err) {
-    warn(`Cannot read ${relPath}: ${err.message}`);
-    return null;
-  }
-
-  const symbols = await parseFileIncremental(cache, filePath, code, engineOpts);
-  if (!symbols) return null;
-
-  stmts.insertNode.run(relPath, 'file', relPath, 0, null);
-
-  for (const def of symbols.definitions) {
-    stmts.insertNode.run(def.name, def.kind, relPath, def.line, def.endLine || null);
-  }
-  for (const exp of symbols.exports) {
-    stmts.insertNode.run(exp.name, exp.kind, relPath, exp.line, null);
-  }
-
-  const newNodes = stmts.countNodes.get(relPath)?.c || 0;
-  const newSymbols = stmts.listSymbols.all(relPath);
-
-  let edgesAdded = 0;
-  const fileNodeRow = stmts.getNodeId.get(relPath, 'file', relPath, 0);
-  if (!fileNodeRow)
-    return { file: relPath, nodesAdded: newNodes, nodesRemoved: oldNodes, edgesAdded: 0 };
-  const fileNodeId = fileNodeRow.id;
-
-  // Load aliases for full import resolution
-  const aliases = { baseUrl: null, paths: {} };
-
-  for (const imp of symbols.imports) {
-    const resolvedPath = resolveImportPath(
-      path.join(rootDir, relPath),
-      imp.source,
-      rootDir,
-      aliases,
-    );
-    const targetRow = stmts.getNodeId.get(resolvedPath, 'file', resolvedPath, 0);
-    if (targetRow) {
-      const edgeKind = imp.reexport ? 'reexports' : imp.typeOnly ? 'imports-type' : 'imports';
-      stmts.insertEdge.run(fileNodeId, targetRow.id, edgeKind, 1.0, 0);
-      edgesAdded++;
-    }
-  }
-
-  const importedNames = new Map();
-  for (const imp of symbols.imports) {
-    const resolvedPath = resolveImportPath(
-      path.join(rootDir, relPath),
-      imp.source,
-      rootDir,
-      aliases,
-    );
-    for (const name of imp.names) {
-      importedNames.set(name.replace(/^\*\s+as\s+/, ''), resolvedPath);
-    }
-  }
-
-  for (const call of symbols.calls) {
-    let caller = null;
-    for (const def of symbols.definitions) {
-      if (def.line <= call.line) {
-        const row = stmts.getNodeId.get(def.name, def.kind, relPath, def.line);
-        if (row) caller = row;
-      }
-    }
-    if (!caller) caller = fileNodeRow;
-
-    const importedFrom = importedNames.get(call.name);
-    let targets;
-    if (importedFrom) {
-      targets = stmts.findNodeInFile.all(call.name, importedFrom);
-    }
-    if (!targets || targets.length === 0) {
-      targets = stmts.findNodeInFile.all(call.name, relPath);
-      if (targets.length === 0) {
-        targets = stmts.findNodeByName.all(call.name);
-      }
-    }
-
-    for (const t of targets) {
-      if (t.id !== caller.id) {
-        stmts.insertEdge.run(
-          caller.id,
-          t.id,
-          'calls',
-          importedFrom ? 1.0 : 0.5,
-          call.dynamic ? 1 : 0,
-        );
-        edgesAdded++;
-      }
-    }
-  }
-
-  const symbolDiff = diffSymbols(oldSymbols, newSymbols);
-  const event = oldNodes === 0 ? 'added' : 'modified';
-
-  return {
-    file: relPath,
-    nodesAdded: newNodes,
-    nodesRemoved: oldNodes,
-    edgesAdded,
-    deleted: false,
-    event,
-    symbolDiff,
-    nodesBefore: oldNodes,
-    nodesAfter: newNodes,
-  };
-}
-
 export async function watchProject(rootDir, opts = {}) {
   const dbPath = path.join(rootDir, '.codegraph', 'graph.db');
   if (!fs.existsSync(dbPath)) {
-    console.error('No graph.db found. Run `codegraph build` first.');
-    process.exit(1);
+    throw new DbError('No graph.db found. Run `codegraph build` first.', { file: dbPath });
   }
 
   const db = openDb(dbPath);
@@ -227,7 +85,9 @@ export async function watchProject(rootDir, opts = {}) {
 
     const results = [];
     for (const filePath of files) {
-      const result = await updateFile(db, rootDir, filePath, stmts, engineOpts, cache);
+      const result = await rebuildFile(db, rootDir, filePath, stmts, engineOpts, cache, {
+        diffSymbols,
+      });
       if (result) results.push(result);
     }
     const updates = results;

package/src/{parser.js → domain/parser.js}

@@ -2,8 +2,8 @@ import fs from 'node:fs';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { Language, Parser, Query } from 'web-tree-sitter';
-import { warn } from './logger.js';
-import { getNative, getNativePackageVersion, loadNative } from './native.js';
+import { warn } from '../infrastructure/logger.js';
+import { getNative, getNativePackageVersion, loadNative } from '../infrastructure/native.js';
 
 // Re-export all extractors for backward compatibility
 export {
@@ -16,7 +16,7 @@ export {
   extractRubySymbols,
   extractRustSymbols,
   extractSymbols,
-} from './extractors/index.js';
+} from '../extractors/index.js';
 
 import {
   extractCSharpSymbols,
@@ -28,12 +28,12 @@ import {
   extractRubySymbols,
   extractRustSymbols,
   extractSymbols,
-} from './extractors/index.js';
+} from '../extractors/index.js';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 function grammarPath(name) {
-  return path.join(__dirname, '..', 'grammars', name);
+  return path.join(__dirname, '..', '..', 'grammars', name);
 }
 
 let _initialized = false;

package/src/domain/queries.js

@@ -0,0 +1,48 @@
+/**
+ * queries.js — Barrel re-export file.
+ *
+ * All query logic lives in the sub-modules under src/analysis/ and src/shared/.
+ * This file exists purely for backward compatibility so that all existing
+ * importers continue to work without changes.
+ */
+
+// ── Re-export from dedicated module for backward compat ───────────────────
+export { isTestFile, TEST_PATTERN } from '../infrastructure/test-filter.js';
+export { iterListFunctions, iterRoles, iterWhere } from '../shared/generators.js';
+// ── Kind/edge constants (canonical source: kinds.js) ─────────────────────
+export {
+  ALL_SYMBOL_KINDS,
+  CORE_EDGE_KINDS,
+  CORE_SYMBOL_KINDS,
+  EVERY_EDGE_KIND,
+  EVERY_SYMBOL_KIND,
+  EXTENDED_SYMBOL_KINDS,
+  STRUCTURAL_EDGE_KINDS,
+  VALID_ROLES,
+} from '../shared/kinds.js';
+// ── Shared utilities ─────────────────────────────────────────────────────
+export { kindIcon, normalizeSymbol } from '../shared/normalize.js';
+export { contextData, explainData } from './analysis/context.js';
+export { fileDepsData, fnDepsData, pathData } from './analysis/dependencies.js';
+export { exportsData } from './analysis/exports.js';
+export {
+  diffImpactData,
+  diffImpactMermaid,
+  fnImpactData,
+  impactAnalysisData,
+} from './analysis/impact.js';
+export {
+  FALSE_POSITIVE_CALLER_THRESHOLD,
+  FALSE_POSITIVE_NAMES,
+  moduleMapData,
+  statsData,
+} from './analysis/module-map.js';
+export { rolesData } from './analysis/roles.js';
+// ── Analysis modules ─────────────────────────────────────────────────────
+export {
+  childrenData,
+  findMatchingNodes,
+  listFunctionsData,
+  queryNameData,
+  whereData,
+} from './analysis/symbol-lookup.js';

package/src/domain/search/generator.js

@@ -0,0 +1,163 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { closeDb, findDbPath, openDb } from '../../db/index.js';
+import { warn } from '../../infrastructure/logger.js';
+import { DbError } from '../../shared/errors.js';
+import { embed, getModelConfig } from './models.js';
+import { buildSourceText } from './strategies/source.js';
+import { buildStructuredText } from './strategies/structured.js';
+
+/**
+ * Rough token estimate (~4 chars per token for code/English).
+ * Conservative — avoids adding a tokenizer dependency.
+ */
+export function estimateTokens(text) {
+  return Math.ceil(text.length / 4);
+}
+
+function initEmbeddingsSchema(db) {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS embeddings (
+      node_id INTEGER PRIMARY KEY,
+      vector BLOB NOT NULL,
+      text_preview TEXT,
+      FOREIGN KEY(node_id) REFERENCES nodes(id)
+    );
+    CREATE TABLE IF NOT EXISTS embedding_meta (
+      key TEXT PRIMARY KEY,
+      value TEXT
+    );
+  `);
+
+  // Add full_text column (idempotent — ignore if already exists)
+  try {
+    db.exec('ALTER TABLE embeddings ADD COLUMN full_text TEXT');
+  } catch {
+    /* column already exists */
+  }
+
+  // FTS5 virtual table for BM25 keyword search
+  db.exec(`
+    CREATE VIRTUAL TABLE IF NOT EXISTS fts_index USING fts5(
+      name,
+      content,
+      tokenize='unicode61'
+    );
+  `);
+}
+
+/**
+ * Build embeddings for all functions/methods/classes in the graph.
+ * @param {string} rootDir - Project root directory
+ * @param {string} modelKey - Model identifier from MODELS registry
+ * @param {string} [customDbPath] - Override path to graph.db
+ * @param {object} [options] - Embedding options
+ * @param {string} [options.strategy='structured'] - 'structured' (graph-enriched) or 'source' (raw code)
+ */
+export async function buildEmbeddings(rootDir, modelKey, customDbPath, options = {}) {
+  const strategy = options.strategy || 'structured';
+  const dbPath = customDbPath || findDbPath(null);
+
+  if (!fs.existsSync(dbPath)) {
+    throw new DbError(
+      `No codegraph database found at ${dbPath}.\nRun "codegraph build" first to analyze your codebase.`,
+      { file: dbPath },
+    );
+  }
+
+  const db = openDb(dbPath);
+  initEmbeddingsSchema(db);
+
+  db.exec('DELETE FROM embeddings');
+  db.exec('DELETE FROM embedding_meta');
+  db.exec('DELETE FROM fts_index');
+
+  const nodes = db
+    .prepare(
+      `SELECT * FROM nodes WHERE kind IN ('function', 'method', 'class') ORDER BY file, line`,
+    )
+    .all();
+
+  console.log(`Building embeddings for ${nodes.length} symbols (strategy: ${strategy})...`);
+
+  const byFile = new Map();
+  for (const node of nodes) {
+    if (!byFile.has(node.file)) byFile.set(node.file, []);
+    byFile.get(node.file).push(node);
+  }
+
+  const texts = [];
+  const nodeIds = [];
+  const nodeNames = [];
+  const previews = [];
+  const config = getModelConfig(modelKey);
+  const contextWindow = config.contextWindow;
+  let overflowCount = 0;
+
+  for (const [file, fileNodes] of byFile) {
+    const fullPath = path.join(rootDir, file);
+    let lines;
+    try {
+      lines = fs.readFileSync(fullPath, 'utf-8').split('\n');
+    } catch (err) {
+      warn(`Cannot read ${file} for embeddings: ${err.message}`);
+      continue;
+    }
+
+    for (const node of fileNodes) {
+      let text =
+        strategy === 'structured'
+          ? buildStructuredText(node, file, lines, db)
+          : buildSourceText(node, file, lines);
+
+      // Detect and handle context window overflow
+      const tokens = estimateTokens(text);
+      if (tokens > contextWindow) {
+        overflowCount++;
+        const maxChars = contextWindow * 4;
+        text = text.slice(0, maxChars);
+      }
+
+      texts.push(text);
+      nodeIds.push(node.id);
+      nodeNames.push(node.name);
+      previews.push(`${node.name} (${node.kind}) -- ${file}:${node.line}`);
+    }
+  }
+
+  if (overflowCount > 0) {
+    warn(
+      `${overflowCount} symbol(s) exceeded model context window (${contextWindow} tokens) and were truncated`,
+    );
+  }
+
+  console.log(`Embedding ${texts.length} symbols...`);
+  const { vectors, dim } = await embed(texts, modelKey);
+
+  const insert = db.prepare(
+    'INSERT OR REPLACE INTO embeddings (node_id, vector, text_preview, full_text) VALUES (?, ?, ?, ?)',
+  );
+  const insertFts = db.prepare('INSERT INTO fts_index(rowid, name, content) VALUES (?, ?, ?)');
+  const insertMeta = db.prepare('INSERT OR REPLACE INTO embedding_meta (key, value) VALUES (?, ?)');
+  const insertAll = db.transaction(() => {
+    for (let i = 0; i < vectors.length; i++) {
+      insert.run(nodeIds[i], Buffer.from(vectors[i].buffer), previews[i], texts[i]);
+      insertFts.run(nodeIds[i], nodeNames[i], texts[i]);
+    }
+    insertMeta.run('model', config.name);
+    insertMeta.run('dim', String(dim));
+    insertMeta.run('count', String(vectors.length));
+    insertMeta.run('fts_count', String(vectors.length));
+    insertMeta.run('strategy', strategy);
+    insertMeta.run('built_at', new Date().toISOString());
+    if (overflowCount > 0) {
+      insertMeta.run('truncated_count', String(overflowCount));
+    }
+  });
+  insertAll();
+
+  console.log(
+    `\nStored ${vectors.length} embeddings (${dim}d, ${config.name}, strategy: ${strategy}) in graph.db`,
+  );
+  closeDb(db);
+}

package/src/domain/search/index.js

@@ -0,0 +1,13 @@
+/**
+ * Embeddings subsystem — public API barrel.
+ *
+ * Re-exports everything consumers previously imported from `../embedder.js`.
+ */
+
+export { buildEmbeddings, estimateTokens } from './generator.js';
+export { DEFAULT_MODEL, disposeModel, EMBEDDING_STRATEGIES, embed, MODELS } from './models.js';
+export { search } from './search/cli-formatter.js';
+export { hybridSearchData } from './search/hybrid.js';
+export { ftsSearchData } from './search/keyword.js';
+export { multiSearchData, searchData } from './search/semantic.js';
+export { cosineSim } from './stores/sqlite-blob.js';

package/src/domain/search/models.js

@@ -0,0 +1,218 @@
+import { execFileSync } from 'node:child_process';
+import { createInterface } from 'node:readline';
+import { info } from '../../infrastructure/logger.js';
+import { ConfigError, EngineError } from '../../shared/errors.js';
+
+// Lazy-load transformers (heavy, optional module)
+let pipeline = null;
+let extractor = null;
+let activeModel = null;
+
+export const MODELS = {
+  minilm: {
+    name: 'Xenova/all-MiniLM-L6-v2',
+    dim: 384,
+    contextWindow: 256,
+    desc: 'Smallest, fastest (~23MB). General text.',
+    quantized: true,
+  },
+  'jina-small': {
+    name: 'Xenova/jina-embeddings-v2-small-en',
+    dim: 512,
+    contextWindow: 8192,
+    desc: 'Small, good quality (~33MB). General text.',
+    quantized: false,
+  },
+  'jina-base': {
+    name: 'Xenova/jina-embeddings-v2-base-en',
+    dim: 768,
+    contextWindow: 8192,
+    desc: 'Good quality (~137MB). General text, 8192 token context.',
+    quantized: false,
+  },
+  'jina-code': {
+    name: 'Xenova/jina-embeddings-v2-base-code',
+    dim: 768,
+    contextWindow: 8192,
+    desc: 'Code-aware (~137MB). Trained on code+text, best for code search.',
+    quantized: false,
+  },
+  nomic: {
+    name: 'Xenova/nomic-embed-text-v1',
+    dim: 768,
+    contextWindow: 8192,
+    desc: 'Good local quality (~137MB). 8192 context.',
+    quantized: false,
+  },
+  'nomic-v1.5': {
+    name: 'nomic-ai/nomic-embed-text-v1.5',
+    dim: 768,
+    contextWindow: 8192,
+    desc: 'Improved nomic (~137MB). Matryoshka dimensions, 8192 context.',
+    quantized: false,
+  },
+  'bge-large': {
+    name: 'Xenova/bge-large-en-v1.5',
+    dim: 1024,
+    contextWindow: 512,
+    desc: 'Best general retrieval (~335MB). Top MTEB scores.',
+    quantized: false,
+  },
+};
+
+export const EMBEDDING_STRATEGIES = ['structured', 'source'];
+
+export const DEFAULT_MODEL = 'nomic-v1.5';
+const BATCH_SIZE_MAP = {
+  minilm: 32,
+  'jina-small': 16,
+  'jina-base': 8,
+  'jina-code': 8,
+  nomic: 8,
+  'nomic-v1.5': 8,
+  'bge-large': 4,
+};
+const DEFAULT_BATCH_SIZE = 32;
+
+/** @internal Used by generator.js — not part of the public barrel. */
+export function getModelConfig(modelKey) {
+  const key = modelKey || DEFAULT_MODEL;
+  const config = MODELS[key];
+  if (!config) {
+    throw new ConfigError(`Unknown model: ${key}. Available: ${Object.keys(MODELS).join(', ')}`);
+  }
+  return config;
+}
+
+/**
+ * Prompt the user to install a missing package interactively.
+ * Returns true if the package was installed, false otherwise.
+ * Skips the prompt entirely in non-TTY environments (CI, piped stdin).
+ * @internal Not part of the public barrel.
+ */
+export function promptInstall(packageName) {
+  if (!process.stdin.isTTY) return Promise.resolve(false);
+
+  return new Promise((resolve) => {
+    const rl = createInterface({ input: process.stdin, output: process.stderr });
+    rl.question(`Semantic search requires ${packageName}. Install it now? [y/N] `, (answer) => {
+      rl.close();
+      if (answer.trim().toLowerCase() !== 'y') return resolve(false);
+      try {
+        execFileSync('npm', ['install', packageName], {
+          stdio: 'inherit',
+          timeout: 300_000,
+        });
+        resolve(true);
+      } catch {
+        resolve(false);
+      }
+    });
+  });
+}
+
+/**
+ * Lazy-load @huggingface/transformers.
+ * If the package is missing, prompts the user to install it interactively.
+ * In non-TTY environments, prints an error and exits.
+ * @internal Not part of the public barrel.
+ */
+export async function loadTransformers() {
+  try {
+    return await import('@huggingface/transformers');
+  } catch {
+    const pkg = '@huggingface/transformers';
+    const installed = await promptInstall(pkg);
+    if (installed) {
+      try {
+        return await import(pkg);
+      } catch (loadErr) {
+        throw new EngineError(
+          `${pkg} was installed but failed to load. Please check your environment.`,
+          { cause: loadErr },
+        );
+      }
+    }
+    throw new EngineError(`Semantic search requires ${pkg}.\nInstall it with: npm install ${pkg}`);
+  }
+}
+
+/**
+ * Dispose the current ONNX session and free memory.
+ * Safe to call when no model is loaded (no-op).
+ */
+export async function disposeModel() {
+  if (extractor) {
+    await extractor.dispose();
+    extractor = null;
+  }
+  activeModel = null;
+}
+
+async function loadModel(modelKey) {
+  const config = getModelConfig(modelKey);
+
+  if (extractor && activeModel === config.name) return { extractor, config };
+
+  // Dispose previous model before loading a different one
+  await disposeModel();
+
+  const transformers = await loadTransformers();
+  pipeline = transformers.pipeline;
+
+  info(`Loading embedding model: ${config.name} (${config.dim}d)...`);
+  const pipelineOpts = config.quantized ? { quantized: true } : {};
+  try {
+    extractor = await pipeline('feature-extraction', config.name, pipelineOpts);
+  } catch (err) {
+    const msg = err.message || String(err);
+    if (msg.includes('Unauthorized') || msg.includes('401') || msg.includes('gated')) {
+      throw new EngineError(
+        `Model "${config.name}" requires authentication.\n` +
+          `This model is gated on HuggingFace and needs an access token.\n\n` +
+          `Options:\n` +
+          `  1. Set HF_TOKEN env var: export HF_TOKEN=hf_...\n` +
+          `  2. Use a public model instead: codegraph embed --model minilm`,
+        { cause: err },
+      );
+    }
+    throw new EngineError(
+      `Failed to load model "${config.name}": ${msg}\n` +
+        `Try a different model: codegraph embed --model minilm`,
+      { cause: err },
+    );
+  }
+  activeModel = config.name;
+  info('Model loaded.');
+  return { extractor, config };
+}
+
+/**
+ * Generate embeddings for an array of texts.
+ */
+export async function embed(texts, modelKey) {
+  const { extractor: ext, config } = await loadModel(modelKey);
+  const dim = config.dim;
+  const results = [];
+  const batchSize = BATCH_SIZE_MAP[modelKey || DEFAULT_MODEL] || DEFAULT_BATCH_SIZE;
+
+  for (let i = 0; i < texts.length; i += batchSize) {
+    const batch = texts.slice(i, i + batchSize);
+    const output = await ext(batch, { pooling: 'mean', normalize: true });
+
+    for (let j = 0; j < batch.length; j++) {
+      const start = j * dim;
+      const vec = new Float32Array(dim);
+      for (let k = 0; k < dim; k++) {
+        vec[k] = output.data[start + k];
+      }
+      results.push(vec);
+    }
+
+    if (texts.length > batchSize) {
+      process.stdout.write(`  Embedded ${Math.min(i + batchSize, texts.length)}/${texts.length}\r`);
+    }
+  }
+
+  return { vectors: results, dim };
+}