@optave/codegraph 3.9.3 → 3.9.5

package/src/domain/parser.ts

@@ -13,6 +13,24 @@ import type {
   LanguageRegistryEntry,
   TypeMapEntry,
 } from '../types.js';
+import { disposeWasmWorkerPool, getWasmWorkerPool } from './wasm-worker-pool.js';
+import type { WorkerAnalysisOpts } from './wasm-worker-protocol.js';
+
+/** Default worker opts: run all analyses so output matches parseFilesFull. */
+const FULL_ANALYSIS: WorkerAnalysisOpts = {
+  ast: true,
+  complexity: true,
+  cfg: true,
+  dataflow: true,
+};
+
+/** Extract-only opts: skip visitor walk for typeMap backfill / similar fast paths. */
+const EXTRACT_ONLY: WorkerAnalysisOpts = {
+  ast: false,
+  complexity: false,
+  cfg: false,
+  dataflow: false,
+};
 
 // Re-export all extractors for backward compatibility
 export {
@@ -262,7 +280,7 @@ function disposeMapEntries(entries: Iterable<[string, any]>, label: string): voi
   }
 }
 
-export function disposeParsers(): void {
+export async function disposeParsers(): Promise<void> {
   if (_cachedParsers) {
     disposeMapEntries(_cachedParsers, 'parser');
     _cachedParsers = null;
@@ -276,6 +294,7 @@ export function disposeParsers(): void {
   _initialized = false;
   _allParsersLoaded = false;
   _loadingPromises.clear();
+  await disposeWasmWorkerPool();
 }
 
 export function getParser(parsers: Map<string, Parser | null>, filePath: string): Parser | null {
@@ -286,33 +305,33 @@ export function getParser(parsers: Map<string, Parser | null>, filePath: string)
 }
 
 /**
- * Pre-parse files missing `_tree` via WASM so downstream phases (CFG, dataflow)
- * don't each need to create parsers and re-parse independently.
- * Only parses files whose extension is in SUPPORTED_EXTENSIONS.
+ * Backfill missing AST-analysis data (astNodes, dataflow, def.complexity,
+ * def.cfg) via the WASM worker pool for files that were parsed by the native
+ * engine but are missing one or more analyses.
+ *
+ * Historically this function populated `symbols._tree` so the main-thread
+ * visitor walk in `ast-analysis/engine.ts` could run. After the worker-isolation
+ * refactor (#965), the worker runs every visitor itself and returns pre-computed
+ * analysis data — `_tree` is never set on the main thread.
+ *
+ * Name is preserved for caller compatibility; the function now ensures
+ * *analysis data* rather than *trees*.
  */
 export async function ensureWasmTrees(
   fileSymbols: Map<string, any>,
   rootDir: string,
 ): Promise<void> {
-  // Single pass: collect absolute paths for files that need parsing
-  const filePaths: string[] = [];
+  // Collect files that still need analysis data and are parseable by WASM.
+  const pending: Array<{ relPath: string; absPath: string; symbols: any }> = [];
   for (const [relPath, symbols] of fileSymbols) {
-    if (!symbols._tree && _extToLang.has(path.extname(relPath).toLowerCase())) {
-      filePaths.push(path.join(rootDir, relPath));
-    }
+    if (symbols._tree) continue; // legacy path — leave existing trees alone
+    if (!_extToLang.has(path.extname(relPath).toLowerCase())) continue;
+    pending.push({ relPath, absPath: path.join(rootDir, relPath), symbols });
   }
-  if (filePaths.length === 0) return;
-  const parsers = await ensureParsersForFiles(filePaths);
+  if (pending.length === 0) return;
 
-  for (const [relPath, symbols] of fileSymbols) {
-    if (symbols._tree) continue;
-    const ext = path.extname(relPath).toLowerCase();
-    const entry = _extToLang.get(ext);
-    if (!entry) continue;
-    const parser = parsers.get(entry.id);
-    if (!parser) continue;
-
-    const absPath = path.join(rootDir, relPath);
+  const pool = getWasmWorkerPool();
+  for (const { relPath, absPath, symbols } of pending) {
     let code: string;
     try {
       code = fs.readFileSync(absPath, 'utf-8');
@@ -320,11 +339,45 @@ export async function ensureWasmTrees(
       debug(`ensureWasmTrees: cannot read ${relPath}: ${(e as Error).message}`);
       continue;
     }
-    try {
-      symbols._tree = parser.parse(code);
-      symbols._langId = entry.id;
-    } catch (e: unknown) {
-      debug(`ensureWasmTrees: parse failed for ${relPath}: ${(e as Error).message}`);
+    const output = await pool.parse(absPath, code, FULL_ANALYSIS);
+    if (!output) continue; // worker crashed or returned null — skip silently
+    mergeAnalysisData(symbols, output);
+  }
+}
+
+/**
+ * Merge pre-computed analysis data from a worker result onto existing symbols.
+ * Only fills gaps — never overwrites fields the caller already populated.
+ * Used to patch native-parsed symbols with worker-produced astNodes / dataflow /
+ * per-definition complexity and cfg.
+ */
+function mergeAnalysisData(symbols: any, worker: ExtractorOutput): void {
+  if (!symbols._langId && worker._langId) symbols._langId = worker._langId;
+  if (!symbols._lineCount && worker._lineCount) symbols._lineCount = worker._lineCount;
+  if (!Array.isArray(symbols.astNodes) && Array.isArray(worker.astNodes)) {
+    symbols.astNodes = worker.astNodes;
+  }
+  if (!symbols.dataflow && worker.dataflow) symbols.dataflow = worker.dataflow;
+  if (worker.typeMap && worker.typeMap.size > 0) {
+    if (!symbols.typeMap || !(symbols.typeMap instanceof Map)) {
+      symbols.typeMap = new Map(worker.typeMap);
+    } else {
+      for (const [k, v] of worker.typeMap) {
+        if (!symbols.typeMap.has(k)) symbols.typeMap.set(k, v);
+      }
+    }
+  }
+  const existingDefs: any[] = Array.isArray(symbols.definitions) ? symbols.definitions : [];
+  const workerDefs: any[] = Array.isArray(worker.definitions) ? worker.definitions : [];
+  // Index existing defs by (kind, name, line) — mirrors engine.ts matching key.
+  const byKey = new Map<string, any>();
+  for (const d of existingDefs) byKey.set(`${d.kind}|${d.name}|${d.line}`, d);
+  for (const wd of workerDefs) {
+    const existing = byKey.get(`${wd.kind}|${wd.name}|${wd.line}`);
+    if (!existing) continue;
+    if (!existing.complexity && wd.complexity) existing.complexity = wd.complexity;
+    if ((!existing.cfg || !Array.isArray(existing.cfg.blocks)) && wd.cfg?.blocks) {
+      existing.cfg = wd.cfg;
     }
   }
 }
@@ -338,6 +391,27 @@ export function isWasmAvailable(): boolean {
   );
 }
 
+/**
+ * Return the set of lowercase file extensions whose WASM grammar is actually
+ * installed on disk. Used to scope engine-parity backfill to files that WASM
+ * can recover — languages without an installed grammar are skipped by both
+ * engines, so they don't represent a native-engine drop.
+ *
+ * Cached on first call; the grammars directory is shipped immutable.
+ */
+let _installedWasmExts: Set<string> | null = null;
+export function getInstalledWasmExtensions(): Set<string> {
+  if (_installedWasmExts) return _installedWasmExts;
+  const exts = new Set<string>();
+  for (const entry of LANGUAGE_REGISTRY) {
+    if (fs.existsSync(grammarPath(entry.grammarFile))) {
+      for (const ext of entry.extensions) exts.add(ext.toLowerCase());
+    }
+  }
+  _installedWasmExts = exts;
+  return exts;
+}
+
 // ── Unified API ──────────────────────────────────────────────────────────────
 
 function resolveEngine(opts: ParseEngineOpts = {}): ResolvedEngine {
@@ -721,23 +795,13 @@ async function backfillTypeMap(
       return { typeMap: new Map(), backfilled: false };
     }
   }
-  const parsers = await ensureParsersForFiles([filePath]);
-  const extracted = wasmExtractSymbols(parsers, filePath, code);
-  try {
-    if (!extracted || extracted.symbols.typeMap.size === 0) {
-      return { typeMap: new Map(), backfilled: false };
-    }
-    return { typeMap: extracted.symbols.typeMap, backfilled: true };
-  } finally {
-    // Free the WASM tree to prevent memory accumulation across repeated builds
-    if (extracted?.tree && typeof extracted.tree.delete === 'function') {
-      try {
-        extracted.tree.delete();
-      } catch (e) {
-        debug(`backfillTypeMap: WASM tree cleanup failed: ${toErrorMessage(e)}`);
-      }
-    }
+  const pool = getWasmWorkerPool();
+  // Extract-only — no visitor walk, we only need the typeMap from this pass.
+  const output = await pool.parse(filePath, code, EXTRACT_ONLY);
+  if (!output || output.typeMap.size === 0) {
+    return { typeMap: new Map(), backfilled: false };
   }
+  return { typeMap: output.typeMap, backfilled: true };
 }
 
 /**
@@ -765,7 +829,16 @@ function wasmExtractSymbols(
   if (!entry) return null;
   const query = _queryCache.get(entry.id) ?? undefined;
   // Query (web-tree-sitter) is structurally compatible with TreeSitterQuery at runtime
-  const symbols = entry.extractor(tree as any, filePath, query as any);
+  let symbols: ExtractorOutput | null;
+  try {
+    symbols = entry.extractor(tree as any, filePath, query as any);
+  } catch (e: unknown) {
+    warn(`Extractor error in ${filePath}: ${(e as Error).message}`);
+    // Free WASM tree to prevent memory leak — web-tree-sitter trees are backed
+    // by WASM linear memory and are not garbage-collected automatically.
+    if (typeof (tree as any).delete === 'function') (tree as any).delete();
+    return null;
+  }
   return symbols ? { symbols, tree, langId: entry.id } : null;
 }
 
@@ -796,10 +869,9 @@ export async function parseFileAuto(
     return patched;
   }
 
-  // WASM path
-  const parsers = await ensureParsersForFiles([filePath]);
-  const extracted = wasmExtractSymbols(parsers, filePath, source);
-  return extracted ? extracted.symbols : null;
+  // WASM path — dispatch to isolated worker
+  const pool = getWasmWorkerPool();
+  return pool.parse(filePath, source, FULL_ANALYSIS);
 }
 
 /** Backfill typeMap via WASM for TS/TSX files parsed by the native engine. */
@@ -812,40 +884,44 @@ async function backfillTypeMapBatch(
   );
   if (tsFiles.length === 0) return;
 
-  const parsers = await ensureParsersForFiles(tsFiles.map((f) => f.filePath));
+  const pool = getWasmWorkerPool();
   for (const { filePath, relPath } of tsFiles) {
-    let extracted: WasmExtractResult | null | undefined;
+    let code: string;
     try {
-      const code = fs.readFileSync(filePath, 'utf-8');
-      extracted = wasmExtractSymbols(parsers, filePath, code);
-      if (extracted?.symbols && extracted.symbols.typeMap.size > 0) {
-        const symbols = result.get(relPath);
-        if (!symbols) continue;
-        symbols.typeMap = extracted.symbols.typeMap;
-        symbols._typeMapBackfilled = true;
-      }
+      code = fs.readFileSync(filePath, 'utf-8');
     } catch (e) {
-      debug(`batchExtract: typeMap backfill failed: ${toErrorMessage(e)}`);
-    } finally {
-      if (extracted?.tree && typeof extracted.tree.delete === 'function') {
-        try {
-          extracted.tree.delete();
-        } catch (e) {
-          debug(`batchExtract: WASM tree cleanup failed: ${toErrorMessage(e)}`);
-        }
-      }
+      debug(`batchExtract: cannot read ${filePath}: ${toErrorMessage(e)}`);
+      continue;
     }
+    const output = await pool.parse(filePath, code, EXTRACT_ONLY);
+    if (!output || output.typeMap.size === 0) continue;
+    const symbols = result.get(relPath);
+    if (!symbols) continue;
+    symbols.typeMap = output.typeMap;
+    symbols._typeMapBackfilled = true;
   }
 }
 
-/** Parse files via WASM engine, returning a Map<relPath, symbols>. */
+/**
+ * Parse files via WASM engine, returning a Map<relPath, symbols>.
+ *
+ * Each file is dispatched to the WASM worker pool. The worker parses, extracts,
+ * and runs all AST analyses (complexity, CFG, dataflow, ast-store) in its own
+ * thread, returning fully pre-computed ExtractorOutput. V8 fatal errors from
+ * tree-sitter WASM (#965) kill only the worker — the pool skips the file and
+ * restarts the worker for the next one.
+ *
+ * `_tree` is NEVER set by this path. All downstream analyses operate on the
+ * pre-computed `astNodes` / `dataflow` / `def.complexity` / `def.cfg` fields.
+ */
 async function parseFilesWasm(
   filePaths: string[],
   rootDir: string,
 ): Promise<Map<string, ExtractorOutput>> {
   const result = new Map<string, ExtractorOutput>();
-  const parsers = await ensureParsersForFiles(filePaths);
+  const pool = getWasmWorkerPool();
   for (const filePath of filePaths) {
+    if (!_extToLang.has(path.extname(filePath).toLowerCase())) continue;
     let code: string;
     try {
       code = fs.readFileSync(filePath, 'utf-8');
@@ -853,13 +929,10 @@ async function parseFilesWasm(
       warn(`Skipping ${path.relative(rootDir, filePath)}: ${(err as Error).message}`);
       continue;
     }
-    const extracted = wasmExtractSymbols(parsers, filePath, code);
-    if (extracted) {
+    const output = await pool.parse(filePath, code, FULL_ANALYSIS);
+    if (output) {
       const relPath = path.relative(rootDir, filePath).split(path.sep).join('/');
-      extracted.symbols._tree = extracted.tree;
-      extracted.symbols._langId = extracted.langId;
-      extracted.symbols._lineCount = code.split('\n').length;
-      result.set(relPath, extracted.symbols);
+      result.set(relPath, output);
     }
   }
   return result;
@@ -884,8 +957,10 @@ export async function parseFilesAuto(
     ? native.parseFilesFull(filePaths, rootDir)
     : native.parseFiles(filePaths, rootDir, true, true);
   const needsTypeMap: { filePath: string; relPath: string }[] = [];
+  const nativeParsed = new Set<string>();
   for (const r of nativeResults) {
     if (!r) continue;
+    nativeParsed.add(r.file);
     const patched = patchNativeResult(r);
     const relPath = path.relative(rootDir, r.file).split(path.sep).join('/');
     result.set(relPath, patched);
@@ -901,6 +976,24 @@ export async function parseFilesAuto(
   if (needsTypeMap.length > 0) {
     await backfillTypeMapBatch(needsTypeMap, result);
   }
+
+  // Engine parity: native may silently drop files whose extensions are in
+  // SUPPORTED_EXTENSIONS (because a WASM grammar exists) but whose Rust
+  // extractor/grammar is missing or fails. WASM handles these — fall back so
+  // both engines process the same file set (#967). Restrict to installed WASM
+  // grammars so we don't warn about files that neither engine can parse.
+  const installedExts = getInstalledWasmExtensions();
+  const dropped = filePaths.filter(
+    (f) => !nativeParsed.has(f) && installedExts.has(path.extname(f).toLowerCase()),
+  );
+  if (dropped.length > 0) {
+    warn(`Native engine dropped ${dropped.length} file(s); falling back to WASM for parity`);
+    const wasmResults = await parseFilesWasm(dropped, rootDir);
+    for (const [relPath, symbols] of wasmResults) {
+      result.set(relPath, symbols);
+    }
+  }
+
   return result;
 }
 

package/src/domain/search/generator.ts

@@ -1,6 +1,6 @@
 import fs from 'node:fs';
 import path from 'node:path';
-import { closeDb, findDbPath, openDb } from '../../db/index.js';
+import { closeDb, findDbPath, getBuildMeta, openDb } from '../../db/index.js';
 import { warn } from '../../infrastructure/logger.js';
 import { DbError } from '../../shared/errors.js';
 import type { BetterSqlite3Database, NodeRow } from '../../types.js';
@@ -73,6 +73,21 @@ export async function buildEmbeddings(
   const db = openDb(dbPath) as BetterSqlite3Database;
   initEmbeddingsSchema(db);
 
+  // Prefer the repo root recorded at build time — embed may be invoked from a
+  // different cwd (e.g. `codegraph embed --db /abs/path/graph.db`) and the
+  // positional rootDir will be wrong in that case. For legacy DBs without
+  // root_dir metadata, fall back to `<dbParent>` only when the DB lives at
+  // the conventional `<root>/.codegraph/graph.db` layout — otherwise trust
+  // the caller-provided rootDir (which may be an explicit positional arg).
+  // `path.dirname(...)` is always non-empty (`'.'` at minimum), so the
+  // conventional-layout check is required to keep the rootDir path reachable.
+  const metaRoot = getBuildMeta(db, 'root_dir');
+  const resolvedDbPath = path.resolve(dbPath);
+  const dbDirName = path.basename(path.dirname(resolvedDbPath));
+  const dbParent =
+    dbDirName === '.codegraph' ? path.dirname(path.dirname(resolvedDbPath)) : undefined;
+  const resolvedRoot = metaRoot || dbParent || rootDir;
+
   db.exec('DELETE FROM embeddings');
   db.exec('DELETE FROM embedding_meta');
   db.exec('DELETE FROM fts_index');
@@ -98,13 +113,17 @@ export async function buildEmbeddings(
   const config = getModelConfig(modelKey);
   const contextWindow = config.contextWindow;
   let overflowCount = 0;
+  let filesRead = 0;
+  let filesSkipped = 0;
 
   for (const [file, fileNodes] of byFile) {
-    const fullPath = path.isAbsolute(file) ? file : path.join(rootDir, file);
+    const fullPath = path.isAbsolute(file) ? file : path.join(resolvedRoot, file);
     let lines: string[];
     try {
       lines = fs.readFileSync(fullPath, 'utf-8').split('\n');
+      filesRead++;
     } catch (err: unknown) {
+      filesSkipped++;
       warn(`Cannot read ${file} for embeddings: ${(err as Error).message}`);
       continue;
     }
@@ -136,6 +155,19 @@ export async function buildEmbeddings(
     );
   }
 
+  // If there were symbols to embed but every file failed to read, the DB was
+  // almost certainly built from a different location than the current cwd.
+  // Surface this clearly instead of emitting a silent "Stored 0 embeddings".
+  if (byFile.size > 0 && filesRead === 0) {
+    closeDb(db);
+    throw new DbError(
+      `embed: could not read any of the ${filesSkipped} source files recorded in the graph — the DB may have been built from a different location than the current working directory.\n` +
+        `Tried resolving against: ${resolvedRoot}\n` +
+        'Pass a positional <dir> argument pointing at the original repo root, or re-run "codegraph build" from that directory.',
+      { file: dbPath },
+    );
+  }
+
   console.log(`Embedding ${texts.length} symbols...`);
   const { vectors, dim } = await embed(texts, modelKey);