@optave/codegraph 3.8.1 → 3.9.0

package/src/domain/analysis/dependencies.ts

@@ -75,14 +75,20 @@ function buildTransitiveCallers(
   let frontier = callers;
 
   for (let d = 2; d <= depth; d++) {
+    // Collect unvisited frontier IDs for a single batched query per depth
+    const unvisited = frontier.filter((f) => !visited.has(f.id));
+    for (const f of unvisited) visited.add(f.id);
+    if (unvisited.length === 0) break;
+
+    const batchCallers = repo.findCallersBatch(unvisited.map((f) => f.id));
     const nextFrontier: typeof frontier = [];
-    for (const f of frontier) {
-      if (visited.has(f.id)) continue;
-      visited.add(f.id);
-      const upstream = repo.findCallers(f.id) as RelatedNodeRow[];
+    const nextFrontierIds = new Set<number>();
+    for (const f of unvisited) {
+      const upstream = batchCallers.get(f.id) || [];
       for (const u of upstream) {
         if (noTests && isTestFile(u.file)) continue;
-        if (!visited.has(u.id)) {
+        if (!visited.has(u.id) && !nextFrontierIds.has(u.id)) {
+          nextFrontierIds.add(u.id);
           nextFrontier.push(u);
         }
       }
@@ -96,7 +102,6 @@ function buildTransitiveCallers(
       }));
     }
     frontier = nextFrontier;
-    if (frontier.length === 0) break;
   }
 
   return transitiveCallers;

package/src/domain/graph/builder/pipeline.ts

@@ -134,7 +134,9 @@ function setupPipeline(ctx: PipelineContext): void {
   // Use NativeDatabase for schema init when native engine is available (Phase 6.13).
   // better-sqlite3 (ctx.db) is still always opened — needed for queries and stages
   // that haven't been migrated to rusqlite yet.
-  const native = loadNative();
+  // Skip native DB entirely when user explicitly requested --engine wasm.
+  const enginePref = ctx.opts.engine || 'auto';
+  const native = enginePref !== 'wasm' ? loadNative() : null;
   if (native?.NativeDatabase) {
     try {
       ctx.nativeDb = native.NativeDatabase.openReadWrite(ctx.dbPath);
@@ -216,6 +218,7 @@ function closeNativeDb(ctx: PipelineContext, label: string): void {
 
 /** Try to reopen the native connection for a given pipeline phase. */
 function reopenNativeDb(ctx: PipelineContext, label: string): void {
+  if ((ctx.opts.engine ?? 'auto') === 'wasm') return;
   const native = loadNative();
   if (!native?.NativeDatabase) return;
   try {
@@ -336,10 +339,14 @@ export async function buildGraph(
     // napi crossings (eliminates WAL dual-connection dance). Falls back
     // to the JS pipeline on failure or when native is unavailable.
     //
-    // Native addon 3.8.0 has a path bug: file_symbols keys are absolute
+    // Native addon ≤3.8.0 has a path bug: file_symbols keys are absolute
     // paths but known_files are relative, causing zero import/call edges.
+    // Native addon ≤3.8.1 has an incremental barrel bug: the Rust pipeline
+    // doesn't re-parse barrel files that are imported by changed files,
+    // causing missing barrel import edges and lost analysis data for
+    // reverse-dep files during incremental builds.
     // Skip the orchestrator for affected versions (fixed in 3.9.0+).
-    const orchestratorBuggy = !!ctx.engineVersion && semverCompare(ctx.engineVersion, '3.8.0') <= 0;
+    const orchestratorBuggy = !!ctx.engineVersion && semverCompare(ctx.engineVersion, '3.8.1') <= 0;
     const forceJs =
       process.env.CODEGRAPH_FORCE_JS_PIPELINE === '1' ||
       ctx.forceFullRebuild ||
@@ -413,18 +420,32 @@ export async function buildGraph(
           `Native build orchestrator completed: ${result.nodeCount ?? 0} nodes, ${result.edgeCount ?? 0} edges, ${result.fileCount ?? 0} files`,
         );
 
-        // ── Run analysis phases (AST, complexity, CFG, dataflow) ──────
-        // Not yet ported to Rust. After the native orchestrator finishes,
-        // reconstruct a minimal fileSymbols map from the DB and run analyses
-        // via the JS engine (native standalone functions + WASM fallback).
+        // ── Run structure + analysis phases after native orchestrator ──
+        // Structure (directory nodes, contains edges, metrics) is not fully
+        // ported to Rust — the native pipeline only handles the small
+        // incremental fast path (≤5 changed files). For full builds and
+        // larger incremental builds, run JS buildStructure() to fill the gap.
+        // Analysis phases (AST, complexity, CFG, dataflow) are also not yet
+        // ported; run via JS engine after reconstructing fileSymbols from DB.
         let analysisTiming = { astMs: 0, complexityMs: 0, cfgMs: 0, dataflowMs: 0 };
+        let structurePatchMs = 0;
         const needsAnalysis =
           opts.ast !== false ||
           opts.complexity !== false ||
           opts.cfg !== false ||
           opts.dataflow !== false;
 
-        if (needsAnalysis) {
+        // The native fast path only runs structure for small incremental
+        // builds: !isFullBuild && changedCount <= 5 && existingFileCount > 20.
+        // For all other cases (full builds, large incrementals), we must
+        // run JS buildStructure() to create directory nodes + contains edges (#804).
+        // Always run JS structure — the native fast-path has an additional
+        // existingFileCount > 20 guard that isn't reflected in the result JSON,
+        // so we can't reliably detect whether native actually ran structure.
+        const nativeHandledStructure = false;
+        const needsStructure = !nativeHandledStructure;
+
+        if (needsAnalysis || needsStructure) {
           // WAL handoff: checkpoint through rusqlite, close nativeDb,
           // reopen better-sqlite3 with a fresh page cache (#715, #736).
           try {
@@ -447,10 +468,8 @@ export async function buildGraph(
           try {
             ctx.db = openDb(ctx.dbPath);
           } catch (reopenErr) {
-            warn(
-              `Failed to reopen DB for analysis after native build: ${(reopenErr as Error).message}`,
-            );
-            // Native build succeeded but we can't run analyses — return partial result
+            warn(`Failed to reopen DB after native build: ${(reopenErr as Error).message}`);
+            // Native build succeeded but we can't run post-processing — return partial result
             return {
               phases: {
                 setupMs: +((p.setupMs ?? 0) + (p.collectMs ?? 0) + (p.detectMs ?? 0)).toFixed(1),
@@ -469,22 +488,14 @@ export async function buildGraph(
             };
           }
 
-          // Reconstruct minimal fileSymbols from DB for analysis visitors.
-          // Each entry needs definitions with name/kind/line/endLine so the
-          // engine can match complexity/CFG results to the right functions.
-          // For incremental builds, scope to only the files that were parsed
-          // in this cycle (matching the JS pipeline's behaviour in run-analyses.ts).
-          const changedFiles = result.changedFiles;
-          let query =
-            'SELECT file, name, kind, line, end_line as endLine FROM nodes WHERE file IS NOT NULL';
-          const params: string[] = [];
-          if (changedFiles && changedFiles.length > 0) {
-            const placeholders = changedFiles.map(() => '?').join(',');
-            query += ` AND file IN (${placeholders})`;
-            params.push(...changedFiles);
-          }
-          query += ' ORDER BY file, line';
-          const rows = ctx.db.prepare(query).all(...params) as {
+          // Reconstruct fileSymbols from DB. For structure we need ALL files
+          // (to build complete directory tree); for analysis we scope to
+          // changed files only. Load all files, then scope analysis later.
+          const allFileRows = ctx.db
+            .prepare(
+              'SELECT file, name, kind, line, end_line as endLine FROM nodes WHERE file IS NOT NULL ORDER BY file, line',
+            )
+            .all() as {
             file: string;
             name: string;
             kind: string;
@@ -492,9 +503,9 @@ export async function buildGraph(
             endLine: number | null;
           }[];
 
-          const fileSymbols = new Map<string, ExtractorOutput>();
-          for (const row of rows) {
-            let entry = fileSymbols.get(row.file);
+          const allFileSymbols = new Map<string, ExtractorOutput>();
+          for (const row of allFileRows) {
+            let entry = allFileSymbols.get(row.file);
             if (!entry) {
               entry = {
                 definitions: [],
@@ -504,7 +515,7 @@ export async function buildGraph(
                 exports: [],
                 typeMap: new Map(),
               };
-              fileSymbols.set(row.file, entry);
+              allFileSymbols.set(row.file, entry);
             }
             entry.definitions.push({
               name: row.name,
@@ -514,45 +525,155 @@ export async function buildGraph(
             });
           }
 
-          // Reopen nativeDb for analysis features (suspend/resume WAL pattern).
-          const native = loadNative();
-          if (native?.NativeDatabase) {
+          // Populate import/export counts from DB edges so buildStructure
+          // computes correct import_count/export_count in node_metrics.
+          // The extractor arrays aren't persisted to the DB, so we derive
+          // counts from edge data instead (#804).
+          const importCountRows = ctx.db
+            .prepare(
+              `SELECT n.file, COUNT(*) AS cnt
+               FROM edges e JOIN nodes n ON e.source_id = n.id
+               WHERE e.kind IN ('imports', 'imports-type', 'dynamic-imports')
+                 AND n.file IS NOT NULL
+               GROUP BY n.file`,
+            )
+            .all() as { file: string; cnt: number }[];
+          for (const row of importCountRows) {
+            const entry = allFileSymbols.get(row.file);
+            if (entry) entry.imports = new Array(row.cnt) as ExtractorOutput['imports'];
+          }
+          // Export count: definitions in this file that are imported by other files
+          const exportCountRows = ctx.db
+            .prepare(
+              `SELECT n_tgt.file, COUNT(DISTINCT n_tgt.id) AS cnt
+               FROM edges e
+               JOIN nodes n_tgt ON e.target_id = n_tgt.id
+               JOIN nodes n_src ON e.source_id = n_src.id
+               WHERE e.kind IN ('imports', 'imports-type', 'reexports')
+                 AND n_tgt.file IS NOT NULL
+                 AND n_src.file != n_tgt.file
+               GROUP BY n_tgt.file`,
+            )
+            .all() as { file: string; cnt: number }[];
+          for (const row of exportCountRows) {
+            const entry = allFileSymbols.get(row.file);
+            if (entry) entry.exports = new Array(row.cnt) as ExtractorOutput['exports'];
+          }
+
+          // ── Structure phase: directory nodes + contains edges (#804) ──
+          if (needsStructure) {
+            const structureStart = performance.now();
             try {
-              ctx.nativeDb = native.NativeDatabase.openReadWrite(ctx.dbPath);
-              if (ctx.engineOpts) ctx.engineOpts.nativeDb = ctx.nativeDb;
-            } catch {
-              ctx.nativeDb = undefined;
-              if (ctx.engineOpts) ctx.engineOpts.nativeDb = undefined;
+              // Derive directories from file paths
+              const directories = new Set<string>();
+              for (const relPath of allFileSymbols.keys()) {
+                const parts = relPath.split('/');
+                for (let i = 1; i < parts.length; i++) {
+                  directories.add(parts.slice(0, i).join('/'));
+                }
+              }
+
+              // Build line count map from DB metrics or file content
+              const lineCountMap = new Map<string, number>();
+              const cachedLineCounts = ctx.db
+                .prepare(
+                  `SELECT n.name AS file, m.line_count
+                 FROM node_metrics m JOIN nodes n ON m.node_id = n.id
+                 WHERE n.kind = 'file'`,
+                )
+                .all() as Array<{ file: string; line_count: number }>;
+              for (const row of cachedLineCounts) {
+                lineCountMap.set(row.file, row.line_count);
+              }
+
+              // Native ran no structure at all — always do a full rebuild so
+              // every directory gets nodes + contains edges (#804).
+              const changedFilePaths = null;
+
+              const { buildStructure: buildStructureFn } = (await import(
+                '../../../features/structure.js'
+              )) as {
+                buildStructure: (
+                  db: typeof ctx.db,
+                  fileSymbols: Map<string, ExtractorOutput>,
+                  rootDir: string,
+                  lineCountMap: Map<string, number>,
+                  directories: Set<string>,
+                  changedFiles: string[] | null,
+                ) => void;
+              };
+              buildStructureFn(
+                ctx.db,
+                allFileSymbols,
+                ctx.rootDir,
+                lineCountMap,
+                directories,
+                changedFilePaths,
+              );
+              debug('Structure phase completed after native orchestrator');
+            } catch (err) {
+              warn(`Structure phase failed after native build: ${toErrorMessage(err)}`);
             }
+            structurePatchMs = performance.now() - structureStart;
           }
 
-          try {
-            const { runAnalyses: runAnalysesFn } = await import('../../../ast-analysis/engine.js');
-            analysisTiming = await runAnalysesFn(
-              ctx.db,
-              fileSymbols,
-              ctx.rootDir,
-              opts,
-              ctx.engineOpts,
-            );
-          } catch (err) {
-            warn(`Analysis phases failed after native build: ${toErrorMessage(err)}`);
-          }
+          // ── Analysis phase ──
+          if (needsAnalysis) {
+            // Scope analysis fileSymbols to changed files only
+            const changedFiles = result.changedFiles;
+            let analysisFileSymbols: Map<string, ExtractorOutput>;
+            if (changedFiles && changedFiles.length > 0) {
+              analysisFileSymbols = new Map();
+              for (const f of changedFiles) {
+                const entry = allFileSymbols.get(f);
+                if (entry) analysisFileSymbols.set(f, entry);
+              }
+            } else {
+              analysisFileSymbols = allFileSymbols;
+            }
 
-          // Close nativeDb after analyses
-          if (ctx.nativeDb) {
-            try {
-              ctx.nativeDb.exec('PRAGMA wal_checkpoint(TRUNCATE)');
-            } catch {
-              /* ignore checkpoint errors */
+            // Reopen nativeDb for analysis features (suspend/resume WAL pattern).
+            const native = loadNative();
+            if (native?.NativeDatabase) {
+              try {
+                ctx.nativeDb = native.NativeDatabase.openReadWrite(ctx.dbPath);
+                if (ctx.engineOpts) ctx.engineOpts.nativeDb = ctx.nativeDb;
+              } catch {
+                ctx.nativeDb = undefined;
+                if (ctx.engineOpts) ctx.engineOpts.nativeDb = undefined;
+              }
             }
+
             try {
-              ctx.nativeDb.close();
-            } catch {
-              /* ignore close errors */
+              const { runAnalyses: runAnalysesFn } = await import(
+                '../../../ast-analysis/engine.js'
+              );
+              analysisTiming = await runAnalysesFn(
+                ctx.db,
+                analysisFileSymbols,
+                ctx.rootDir,
+                opts,
+                ctx.engineOpts,
+              );
+            } catch (err) {
+              warn(`Analysis phases failed after native build: ${toErrorMessage(err)}`);
+            }
+
+            // Close nativeDb after analyses
+            if (ctx.nativeDb) {
+              try {
+                ctx.nativeDb.exec('PRAGMA wal_checkpoint(TRUNCATE)');
+              } catch {
+                /* ignore checkpoint errors */
+              }
+              try {
+                ctx.nativeDb.close();
+              } catch {
+                /* ignore close errors */
+              }
+              ctx.nativeDb = undefined;
+              if (ctx.engineOpts) ctx.engineOpts.nativeDb = undefined;
             }
-            ctx.nativeDb = undefined;
-            if (ctx.engineOpts) ctx.engineOpts.nativeDb = undefined;
           }
         }
 
@@ -564,7 +685,7 @@ export async function buildGraph(
             insertMs: +(p.insertMs ?? 0).toFixed(1),
             resolveMs: +(p.resolveMs ?? 0).toFixed(1),
             edgesMs: +(p.edgesMs ?? 0).toFixed(1),
-            structureMs: +(p.structureMs ?? 0).toFixed(1),
+            structureMs: +((p.structureMs ?? 0) + structurePatchMs).toFixed(1),
             rolesMs: +(p.rolesMs ?? 0).toFixed(1),
             astMs: +(analysisTiming.astMs ?? 0).toFixed(1),
             complexityMs: +(analysisTiming.complexityMs ?? 0).toFixed(1),

package/src/domain/graph/builder/stages/build-edges.ts

@@ -354,7 +354,10 @@ function buildImportedNamesForNative(
   rootDir: string,
 ): Array<{ name: string; file: string }> {
   const importedNames: Array<{ name: string; file: string }> = [];
-  for (const imp of symbols.imports) {
+  // Process dynamic imports first (lower priority), then static imports
+  // (higher priority). Rust HashMap::collect keeps the last entry per key,
+  // so static imports win when both contribute the same name.
+  const addImports = (imp: (typeof symbols.imports)[number]) => {
     const resolvedPath = getResolved(ctx, path.join(rootDir, relPath), imp.source);
     for (const name of imp.names) {
       const cleanName = name.replace(/^\*\s+as\s+/, '');
@@ -365,6 +368,12 @@ function buildImportedNamesForNative(
       }
       importedNames.push({ name: cleanName, file: targetFile });
     }
+  };
+  for (const imp of symbols.imports) {
+    if (imp.dynamicImport) addImports(imp);
+  }
+  for (const imp of symbols.imports) {
+    if (!imp.dynamicImport) addImports(imp);
   }
   return importedNames;
 }
@@ -409,12 +418,25 @@ function buildImportedNamesMap(
   rootDir: string,
 ): Map<string, string> {
   const importedNames = new Map<string, string>();
+  // Process dynamic imports first (lower priority), then static imports
+  // (higher priority). Static imports represent direct bindings while dynamic
+  // imports often use aliased destructuring (`{ foo: bar } = await import(…)`).
+  // When both contribute the same name, the static binding is authoritative.
   for (const imp of symbols.imports) {
+    if (!imp.dynamicImport) continue;
     const resolvedPath = getResolved(ctx, path.join(rootDir, relPath), imp.source);
     for (const name of imp.names) {
       importedNames.set(name.replace(/^\*\s+as\s+/, ''), resolvedPath);
     }
   }
+  for (const imp of symbols.imports) {
+    if (imp.dynamicImport) continue;
+    const resolvedPath = getResolved(ctx, path.join(rootDir, relPath), imp.source);
+    for (const name of imp.names) {
+      const cleanName = name.replace(/^\*\s+as\s+/, '');
+      importedNames.set(cleanName, resolvedPath);
+    }
+  }
   return importedNames;
 }