@optave/codegraph 3.1.3 → 3.1.4

package/src/builder/context.js

@@ -0,0 +1,85 @@
+/**
+ * PipelineContext — shared mutable state threaded through all build stages.
+ *
+ * Each stage reads what it needs and writes what it produces.
+ * This replaces the closure-captured locals in the old monolithic buildGraph().
+ */
+export class PipelineContext {
+  // ── Inputs (set during setup) ──────────────────────────────────────
+  /** @type {string} Absolute root directory */
+  rootDir;
+  /** @type {import('better-sqlite3').Database} */
+  db;
+  /** @type {string} Absolute path to the database file */
+  dbPath;
+  /** @type {object} From loadConfig() */
+  config;
+  /** @type {object} Original buildGraph opts */
+  opts;
+  /** @type {{ engine: string, dataflow: boolean, ast: boolean }} */
+  engineOpts;
+  /** @type {string} 'native' | 'wasm' */
+  engineName;
+  /** @type {string|null} */
+  engineVersion;
+  /** @type {{ baseUrl: string|null, paths: object }} */
+  aliases;
+  /** @type {boolean} Whether incremental mode is enabled */
+  incremental;
+  /** @type {boolean} Force full rebuild (engine/schema mismatch) */
+  forceFullRebuild = false;
+  /** @type {number} Current schema version */
+  schemaVersion;
+
+  // ── File collection (set by collectFiles stage) ────────────────────
+  /** @type {string[]} Absolute file paths */
+  allFiles;
+  /** @type {Set<string>} Absolute directory paths */
+  discoveredDirs;
+
+  // ── Change detection (set by detectChanges stage) ──────────────────
+  /** @type {boolean} */
+  isFullBuild;
+  /** @type {Array<{ file: string, relPath?: string, content?: string, hash?: string, stat?: object, _reverseDepOnly?: boolean }>} */
+  parseChanges;
+  /** @type {Array<{ relPath: string, hash: string, stat: object }>} Metadata-only self-heal updates */
+  metadataUpdates;
+  /** @type {string[]} Relative paths of deleted files */
+  removed;
+  /** @type {boolean} True when no changes detected — skip remaining stages */
+  earlyExit = false;
+
+  // ── Parsing (set by parseFiles stage) ──────────────────────────────
+  /** @type {Map<string, object>} relPath → symbols from parseFilesAuto */
+  allSymbols;
+  /** @type {Map<string, object>} relPath → symbols (includes incrementally loaded) */
+  fileSymbols;
+  /** @type {Array<{ file: string, relPath?: string }>} Files to parse this build */
+  filesToParse;
+
+  // ── Import resolution (set by resolveImports stage) ────────────────
+  /** @type {Map<string, string>|null} "absFile|source" → resolved path */
+  batchResolved;
+  /** @type {Map<string, Array>} relPath → re-export descriptors */
+  reexportMap;
+  /** @type {Set<string>} Files loaded only for barrel resolution (don't rebuild edges) */
+  barrelOnlyFiles;
+
+  // ── Node lookup (set by insertNodes / buildEdges stages) ───────────
+  /** @type {Map<string, Array>} name → node rows */
+  nodesByName;
+  /** @type {Map<string, Array>} "name|file" → node rows */
+  nodesByNameAndFile;
+
+  // ── Misc state ─────────────────────────────────────────────────────
+  /** @type {boolean} Whether embeddings table exists */
+  hasEmbeddings = false;
+  /** @type {Map<string, number>} relPath → line count */
+  lineCountMap;
+
+  // ── Phase timing ───────────────────────────────────────────────────
+  timing = {};
+
+  /** @type {number} performance.now() at build start */
+  buildStart;
+}

package/src/builder/helpers.js

@@ -0,0 +1,218 @@
+/**
+ * Builder helper functions — shared utilities used across pipeline stages.
+ *
+ * Extracted from the monolithic builder.js so stages can import individually.
+ */
+import { createHash } from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+import { EXTENSIONS, IGNORE_DIRS } from '../constants.js';
+import { purgeFilesData } from '../db.js';
+import { warn } from '../logger.js';
+
+export const BUILTIN_RECEIVERS = new Set([
+  'console',
+  'Math',
+  'JSON',
+  'Object',
+  'Array',
+  'String',
+  'Number',
+  'Boolean',
+  'Date',
+  'RegExp',
+  'Map',
+  'Set',
+  'WeakMap',
+  'WeakSet',
+  'Promise',
+  'Symbol',
+  'Error',
+  'TypeError',
+  'RangeError',
+  'Proxy',
+  'Reflect',
+  'Intl',
+  'globalThis',
+  'window',
+  'document',
+  'process',
+  'Buffer',
+  'require',
+]);
+
+/**
+ * Recursively collect all source files under `dir`.
+ * When `directories` is a Set, also tracks which directories contain files.
+ */
+export function collectFiles(
+  dir,
+  files = [],
+  config = {},
+  directories = null,
+  _visited = new Set(),
+) {
+  const trackDirs = directories instanceof Set;
+  let hasFiles = false;
+
+  // Merge config ignoreDirs with defaults
+  const extraIgnore = config.ignoreDirs ? new Set(config.ignoreDirs) : null;
+
+  // Detect symlink loops (before I/O to avoid wasted readdirSync)
+  let realDir;
+  try {
+    realDir = fs.realpathSync(dir);
+  } catch {
+    return trackDirs ? { files, directories } : files;
+  }
+  if (_visited.has(realDir)) {
+    warn(`Symlink loop detected, skipping: ${dir}`);
+    return trackDirs ? { files, directories } : files;
+  }
+  _visited.add(realDir);
+
+  let entries;
+  try {
+    entries = fs.readdirSync(dir, { withFileTypes: true });
+  } catch (err) {
+    warn(`Cannot read directory ${dir}: ${err.message}`);
+    return trackDirs ? { files, directories } : files;
+  }
+
+  for (const entry of entries) {
+    if (entry.name.startsWith('.') && entry.name !== '.') {
+      if (IGNORE_DIRS.has(entry.name)) continue;
+      if (entry.isDirectory()) continue;
+    }
+    if (IGNORE_DIRS.has(entry.name)) continue;
+    if (extraIgnore?.has(entry.name)) continue;
+
+    const full = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      collectFiles(full, files, config, directories, _visited);
+    } else if (EXTENSIONS.has(path.extname(entry.name))) {
+      files.push(full);
+      hasFiles = true;
+    }
+  }
+  if (trackDirs && hasFiles) {
+    directories.add(dir);
+  }
+  return trackDirs ? { files, directories } : files;
+}
+
+/**
+ * Load path aliases from tsconfig.json / jsconfig.json.
+ */
+export function loadPathAliases(rootDir) {
+  const aliases = { baseUrl: null, paths: {} };
+  for (const configName of ['tsconfig.json', 'jsconfig.json']) {
+    const configPath = path.join(rootDir, configName);
+    if (!fs.existsSync(configPath)) continue;
+    try {
+      const raw = fs
+        .readFileSync(configPath, 'utf-8')
+        .replace(/\/\/.*$/gm, '')
+        .replace(/\/\*[\s\S]*?\*\//g, '')
+        .replace(/,\s*([\]}])/g, '$1');
+      const config = JSON.parse(raw);
+      const opts = config.compilerOptions || {};
+      if (opts.baseUrl) aliases.baseUrl = path.resolve(rootDir, opts.baseUrl);
+      if (opts.paths) {
+        for (const [pattern, targets] of Object.entries(opts.paths)) {
+          aliases.paths[pattern] = targets.map((t) => path.resolve(aliases.baseUrl || rootDir, t));
+        }
+      }
+      break;
+    } catch (err) {
+      warn(`Failed to parse ${configName}: ${err.message}`);
+    }
+  }
+  return aliases;
+}
+
+/**
+ * Compute MD5 hash of file contents for incremental builds.
+ */
+export function fileHash(content) {
+  return createHash('md5').update(content).digest('hex');
+}
+
+/**
+ * Stat a file, returning { mtimeMs, size } or null on error.
+ */
+export function fileStat(filePath) {
+  try {
+    const s = fs.statSync(filePath);
+    return { mtimeMs: s.mtimeMs, size: s.size };
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Read a file with retry on transient errors (EBUSY/EACCES/EPERM).
+ */
+const TRANSIENT_CODES = new Set(['EBUSY', 'EACCES', 'EPERM']);
+const RETRY_DELAY_MS = 50;
+
+export function readFileSafe(filePath, retries = 2) {
+  for (let attempt = 0; ; attempt++) {
+    try {
+      return fs.readFileSync(filePath, 'utf-8');
+    } catch (err) {
+      if (attempt < retries && TRANSIENT_CODES.has(err.code)) {
+        const end = Date.now() + RETRY_DELAY_MS;
+        while (Date.now() < end) {}
+        continue;
+      }
+      throw err;
+    }
+  }
+}
+
+/**
+ * Purge all graph data for the specified files.
+ */
+export function purgeFilesFromGraph(db, files, options = {}) {
+  purgeFilesData(db, files, options);
+}
+
+/** Batch INSERT chunk size for multi-value INSERTs. */
+export const BATCH_CHUNK = 200;
+
+/**
+ * Batch-insert node rows via multi-value INSERT statements.
+ * Each row: [name, kind, file, line, end_line, parent_id, qualified_name, scope, visibility]
+ */
+export function batchInsertNodes(db, rows) {
+  if (!rows.length) return;
+  const ph = '(?,?,?,?,?,?,?,?,?)';
+  for (let i = 0; i < rows.length; i += BATCH_CHUNK) {
+    const chunk = rows.slice(i, i + BATCH_CHUNK);
+    const vals = [];
+    for (const r of chunk) vals.push(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8]);
+    db.prepare(
+      'INSERT OR IGNORE INTO nodes (name,kind,file,line,end_line,parent_id,qualified_name,scope,visibility) VALUES ' +
+        chunk.map(() => ph).join(','),
+    ).run(...vals);
+  }
+}
+
+/**
+ * Batch-insert edge rows via multi-value INSERT statements.
+ * Each row: [source_id, target_id, kind, confidence, dynamic]
+ */
+export function batchInsertEdges(db, rows) {
+  if (!rows.length) return;
+  const ph = '(?,?,?,?,?)';
+  for (let i = 0; i < rows.length; i += BATCH_CHUNK) {
+    const chunk = rows.slice(i, i + BATCH_CHUNK);
+    const vals = [];
+    for (const r of chunk) vals.push(r[0], r[1], r[2], r[3], r[4]);
+    db.prepare(
+      'INSERT INTO edges (source_id,target_id,kind,confidence,dynamic) VALUES ' +
+        chunk.map(() => ph).join(','),
+    ).run(...vals);
+  }
+}

package/src/builder/incremental.js

@@ -0,0 +1,178 @@
+/**
+ * Incremental single-file rebuild — used by watch mode.
+ *
+ * Reuses pipeline helpers instead of duplicating node insertion and edge building
+ * logic from the main builder. This eliminates the watcher.js divergence (ROADMAP 3.9).
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { normalizePath } from '../constants.js';
+import { warn } from '../logger.js';
+import { parseFileIncremental } from '../parser.js';
+import { computeConfidence, resolveImportPath } from '../resolve.js';
+import { BUILTIN_RECEIVERS, readFileSafe } from './helpers.js';
+
+/**
+ * Parse a single file and update the database incrementally.
+ *
+ * @param {import('better-sqlite3').Database} db
+ * @param {string} rootDir - Absolute root directory
+ * @param {string} filePath - Absolute file path
+ * @param {object} stmts - Prepared DB statements
+ * @param {object} engineOpts - Engine options
+ * @param {object|null} cache - Parse tree cache (native only)
+ * @param {object} [options]
+ * @param {Function} [options.diffSymbols] - Symbol diff function
+ * @returns {Promise<object|null>} Update result or null on failure
+ */
+export async function rebuildFile(_db, rootDir, filePath, stmts, engineOpts, cache, options = {}) {
+  const { diffSymbols } = options;
+  const relPath = normalizePath(path.relative(rootDir, filePath));
+  const oldNodes = stmts.countNodes.get(relPath)?.c || 0;
+  const oldSymbols = diffSymbols ? stmts.listSymbols.all(relPath) : [];
+
+  stmts.deleteEdgesForFile.run(relPath);
+  stmts.deleteNodes.run(relPath);
+
+  if (!fs.existsSync(filePath)) {
+    if (cache) cache.remove(filePath);
+    const symbolDiff = diffSymbols ? diffSymbols(oldSymbols, []) : null;
+    return {
+      file: relPath,
+      nodesAdded: 0,
+      nodesRemoved: oldNodes,
+      edgesAdded: 0,
+      deleted: true,
+      event: 'deleted',
+      symbolDiff,
+      nodesBefore: oldNodes,
+      nodesAfter: 0,
+    };
+  }
+
+  let code;
+  try {
+    code = readFileSafe(filePath);
+  } catch (err) {
+    warn(`Cannot read ${relPath}: ${err.message}`);
+    return null;
+  }
+
+  const symbols = await parseFileIncremental(cache, filePath, code, engineOpts);
+  if (!symbols) return null;
+
+  // Insert nodes
+  stmts.insertNode.run(relPath, 'file', relPath, 0, null);
+  for (const def of symbols.definitions) {
+    stmts.insertNode.run(def.name, def.kind, relPath, def.line, def.endLine || null);
+  }
+  for (const exp of symbols.exports) {
+    stmts.insertNode.run(exp.name, exp.kind, relPath, exp.line, null);
+  }
+
+  const newNodes = stmts.countNodes.get(relPath)?.c || 0;
+  const newSymbols = diffSymbols ? stmts.listSymbols.all(relPath) : [];
+
+  let edgesAdded = 0;
+  const fileNodeRow = stmts.getNodeId.get(relPath, 'file', relPath, 0);
+  if (!fileNodeRow)
+    return { file: relPath, nodesAdded: newNodes, nodesRemoved: oldNodes, edgesAdded: 0 };
+  const fileNodeId = fileNodeRow.id;
+
+  // Load aliases for import resolution
+  const aliases = { baseUrl: null, paths: {} };
+
+  // Import edges
+  for (const imp of symbols.imports) {
+    const resolvedPath = resolveImportPath(
+      path.join(rootDir, relPath),
+      imp.source,
+      rootDir,
+      aliases,
+    );
+    const targetRow = stmts.getNodeId.get(resolvedPath, 'file', resolvedPath, 0);
+    if (targetRow) {
+      const edgeKind = imp.reexport ? 'reexports' : imp.typeOnly ? 'imports-type' : 'imports';
+      stmts.insertEdge.run(fileNodeId, targetRow.id, edgeKind, 1.0, 0);
+      edgesAdded++;
+    }
+  }
+
+  // Build import name → resolved file mapping
+  const importedNames = new Map();
+  for (const imp of symbols.imports) {
+    const resolvedPath = resolveImportPath(
+      path.join(rootDir, relPath),
+      imp.source,
+      rootDir,
+      aliases,
+    );
+    for (const name of imp.names) {
+      importedNames.set(name.replace(/^\*\s+as\s+/, ''), resolvedPath);
+    }
+  }
+
+  // Call edges
+  for (const call of symbols.calls) {
+    if (call.receiver && BUILTIN_RECEIVERS.has(call.receiver)) continue;
+
+    let caller = null;
+    let callerSpan = Infinity;
+    for (const def of symbols.definitions) {
+      if (def.line <= call.line) {
+        const end = def.endLine || Infinity;
+        if (call.line <= end) {
+          const span = end - def.line;
+          if (span < callerSpan) {
+            const row = stmts.getNodeId.get(def.name, def.kind, relPath, def.line);
+            if (row) {
+              caller = row;
+              callerSpan = span;
+            }
+          }
+        } else if (!caller) {
+          const row = stmts.getNodeId.get(def.name, def.kind, relPath, def.line);
+          if (row) caller = row;
+        }
+      }
+    }
+    if (!caller) caller = fileNodeRow;
+
+    const importedFrom = importedNames.get(call.name);
+    let targets;
+    if (importedFrom) {
+      targets = stmts.findNodeInFile.all(call.name, importedFrom);
+    }
+    if (!targets || targets.length === 0) {
+      targets = stmts.findNodeInFile.all(call.name, relPath);
+      if (targets.length === 0) {
+        targets = stmts.findNodeByName.all(call.name);
+      }
+    }
+
+    for (const t of targets) {
+      if (t.id !== caller.id) {
+        const confidence = importedFrom
+          ? computeConfidence(relPath, t.file, importedFrom)
+          : computeConfidence(relPath, t.file, null);
+        stmts.insertEdge.run(caller.id, t.id, 'calls', confidence, call.dynamic ? 1 : 0);
+        edgesAdded++;
+      }
+    }
+  }
+
+  const symbolDiff = diffSymbols ? diffSymbols(oldSymbols, newSymbols) : null;
+  const event = oldNodes === 0 ? 'added' : 'modified';
+
+  return {
+    file: relPath,
+    nodesAdded: newNodes,
+    nodesRemoved: oldNodes,
+    edgesAdded,
+    deleted: false,
+    event,
+    symbolDiff,
+    nodesBefore: oldNodes,
+    nodesAfter: newNodes,
+  };
+}

package/src/builder/pipeline.js

@@ -0,0 +1,130 @@
+/**
+ * Pipeline orchestrator — runs build stages sequentially through a shared PipelineContext.
+ *
+ * This is the heart of the builder refactor (ROADMAP 3.9): the monolithic buildGraph()
+ * is decomposed into independently testable stages that communicate via PipelineContext.
+ */
+import path from 'node:path';
+import { performance } from 'node:perf_hooks';
+import { loadConfig } from '../config.js';
+import { closeDb, getBuildMeta, initSchema, MIGRATIONS, openDb } from '../db.js';
+import { info } from '../logger.js';
+import { getActiveEngine } from '../parser.js';
+import { PipelineContext } from './context.js';
+import { loadPathAliases } from './helpers.js';
+import { buildEdges } from './stages/build-edges.js';
+import { buildStructure } from './stages/build-structure.js';
+// Pipeline stages
+import { collectFiles } from './stages/collect-files.js';
+import { detectChanges } from './stages/detect-changes.js';
+import { finalize } from './stages/finalize.js';
+import { insertNodes } from './stages/insert-nodes.js';
+import { parseFiles } from './stages/parse-files.js';
+import { resolveImports } from './stages/resolve-imports.js';
+import { runAnalyses } from './stages/run-analyses.js';
+
+/**
+ * Build the dependency graph for a codebase.
+ *
+ * Signature and return value are identical to the original monolithic buildGraph().
+ *
+ * @param {string} rootDir - Root directory to scan
+ * @param {object} [opts] - Build options
+ * @returns {Promise<{ phases: object } | undefined>}
+ */
+export async function buildGraph(rootDir, opts = {}) {
+  const ctx = new PipelineContext();
+  ctx.buildStart = performance.now();
+  ctx.opts = opts;
+
+  // ── Setup (creates DB, loads config, selects engine) ──────────────
+  ctx.rootDir = path.resolve(rootDir);
+  ctx.dbPath = path.join(ctx.rootDir, '.codegraph', 'graph.db');
+  ctx.db = openDb(ctx.dbPath);
+  try {
+    initSchema(ctx.db);
+
+    ctx.config = loadConfig(ctx.rootDir);
+    ctx.incremental =
+      opts.incremental !== false && ctx.config.build && ctx.config.build.incremental !== false;
+
+    ctx.engineOpts = {
+      engine: opts.engine || 'auto',
+      dataflow: opts.dataflow !== false,
+      ast: opts.ast !== false,
+    };
+    const { name: engineName, version: engineVersion } = getActiveEngine(ctx.engineOpts);
+    ctx.engineName = engineName;
+    ctx.engineVersion = engineVersion;
+    info(`Using ${engineName} engine${engineVersion ? ` (v${engineVersion})` : ''}`);
+
+    // Engine/schema mismatch detection
+    ctx.schemaVersion = MIGRATIONS[MIGRATIONS.length - 1].version;
+    ctx.forceFullRebuild = false;
+    if (ctx.incremental) {
+      const prevEngine = getBuildMeta(ctx.db, 'engine');
+      if (prevEngine && prevEngine !== engineName) {
+        info(`Engine changed (${prevEngine} → ${engineName}), promoting to full rebuild.`);
+        ctx.forceFullRebuild = true;
+      }
+      const prevSchema = getBuildMeta(ctx.db, 'schema_version');
+      if (prevSchema && Number(prevSchema) !== ctx.schemaVersion) {
+        info(
+          `Schema version changed (${prevSchema} → ${ctx.schemaVersion}), promoting to full rebuild.`,
+        );
+        ctx.forceFullRebuild = true;
+      }
+    }
+
+    // Path aliases
+    ctx.aliases = loadPathAliases(ctx.rootDir);
+    if (ctx.config.aliases) {
+      for (const [key, value] of Object.entries(ctx.config.aliases)) {
+        const pattern = key.endsWith('/') ? `${key}*` : key;
+        const target = path.resolve(ctx.rootDir, value);
+        ctx.aliases.paths[pattern] = [target.endsWith('/') ? `${target}*` : `${target}/*`];
+      }
+    }
+    if (ctx.aliases.baseUrl || Object.keys(ctx.aliases.paths).length > 0) {
+      info(
+        `Loaded path aliases: baseUrl=${ctx.aliases.baseUrl || 'none'}, ${Object.keys(ctx.aliases.paths).length} path mappings`,
+      );
+    }
+
+    ctx.timing.setupMs = performance.now() - ctx.buildStart;
+
+    // ── Pipeline stages ─────────────────────────────────────────────
+    await collectFiles(ctx);
+    await detectChanges(ctx);
+
+    if (ctx.earlyExit) return;
+
+    await parseFiles(ctx);
+    await insertNodes(ctx);
+    await resolveImports(ctx);
+    await buildEdges(ctx);
+    await buildStructure(ctx);
+    await runAnalyses(ctx);
+    await finalize(ctx);
+  } catch (err) {
+    if (!ctx.earlyExit) closeDb(ctx.db);
+    throw err;
+  }
+
+  return {
+    phases: {
+      setupMs: +ctx.timing.setupMs.toFixed(1),
+      parseMs: +(ctx.timing.parseMs ?? 0).toFixed(1),
+      insertMs: +(ctx.timing.insertMs ?? 0).toFixed(1),
+      resolveMs: +(ctx.timing.resolveMs ?? 0).toFixed(1),
+      edgesMs: +(ctx.timing.edgesMs ?? 0).toFixed(1),
+      structureMs: +(ctx.timing.structureMs ?? 0).toFixed(1),
+      rolesMs: +(ctx.timing.rolesMs ?? 0).toFixed(1),
+      astMs: +(ctx.timing.astMs ?? 0).toFixed(1),
+      complexityMs: +(ctx.timing.complexityMs ?? 0).toFixed(1),
+      ...(ctx.timing.cfgMs != null && { cfgMs: +ctx.timing.cfgMs.toFixed(1) }),
+      ...(ctx.timing.dataflowMs != null && { dataflowMs: +ctx.timing.dataflowMs.toFixed(1) }),
+      finalizeMs: +(ctx.timing.finalizeMs ?? 0).toFixed(1),
+    },
+  };
+}