@optave/codegraph 3.8.0 → 3.9.0

package/src/domain/parser.ts

@@ -102,6 +102,12 @@ let _cachedLanguages: Map<string, Language> | null = null;
 // Query cache for JS/TS/TSX extractors (populated during createParsers)
 const _queryCache: Map<string, Query> = new Map();
 
+// Tracks whether ALL grammars have been loaded (vs. a lazy subset)
+let _allParsersLoaded: boolean = false;
+
+// In-flight grammar loads keyed by language id — prevents concurrent duplicate loads
+const _loadingPromises: Map<string, Promise<void>> = new Map();
+
 // Extensions that need typeMap backfill (type annotations only exist in TS/TSX)
 const TS_BACKFILL_EXTS = new Set(['.ts', '.tsx']);
 
@@ -150,42 +156,87 @@ const TS_EXTRA_PATTERNS: string[] = [
   '(type_alias_declaration name: (type_identifier) @type_name) @type_node',
 ];
 
-export async function createParsers(): Promise<Map<string, Parser | null>> {
-  if (_cachedParsers) return _cachedParsers;
+/**
+ * Load a single language grammar and cache the parser + language + query.
+ * Uses in-flight deduplication so concurrent callers awaiting the same grammar
+ * share a single load rather than producing orphaned WASM instances.
+ * Assumes Parser.init() has already been called and _cachedParsers/_cachedLanguages exist.
+ */
+async function loadLanguage(entry: LanguageRegistryEntry): Promise<void> {
+  if (_cachedParsers!.has(entry.id)) return;
+  const inflight = _loadingPromises.get(entry.id);
+  if (inflight) return inflight;
+  const p = doLoadLanguage(entry).finally(() => _loadingPromises.delete(entry.id));
+  _loadingPromises.set(entry.id, p);
+  return p;
+}
+
+async function doLoadLanguage(entry: LanguageRegistryEntry): Promise<void> {
+  try {
+    const lang = await Language.load(grammarPath(entry.grammarFile));
+    const parser = new Parser();
+    parser.setLanguage(lang);
+    _cachedParsers!.set(entry.id, parser);
+    _cachedLanguages!.set(entry.id, lang);
+    if (entry.extractor === extractSymbols && !_queryCache.has(entry.id)) {
+      const isTS = entry.id === 'typescript' || entry.id === 'tsx';
+      const patterns = isTS
+        ? [...COMMON_QUERY_PATTERNS, ...TS_EXTRA_PATTERNS]
+        : [...COMMON_QUERY_PATTERNS, JS_CLASS_PATTERN];
+      _queryCache.set(entry.id, new Query(lang, patterns.join('\n')));
+    }
+  } catch (e: unknown) {
+    if (entry.required) throw e;
+    warn(
+      `${entry.id} parser failed to initialize: ${(e as Error).message}. ${entry.id} files will be skipped.`,
+    );
+    _cachedParsers!.set(entry.id, null);
+  }
+}
 
+async function initParserRuntime(): Promise<void> {
   if (!_initialized) {
     await Parser.init();
     _initialized = true;
   }
+  if (!_cachedParsers) _cachedParsers = new Map();
+  if (!_cachedLanguages) _cachedLanguages = new Map();
+}
 
-  const parsers = new Map<string, Parser | null>();
-  const languages = new Map<string, Language>();
+/**
+ * Load only the WASM grammars needed for the given file paths.
+ * Grammars already in cache are reused. This avoids the ~500ms cold-start
+ * penalty of loading all 23+ grammars when only 1-2 are needed (e.g. incremental rebuilds).
+ */
+async function ensureParsersForFiles(filePaths: string[]): Promise<Map<string, Parser | null>> {
+  await initParserRuntime();
+  const needed = new Set<LanguageRegistryEntry>();
+  for (const fp of filePaths) {
+    const ext = path.extname(fp).toLowerCase();
+    const entry = _extToLang.get(ext);
+    if (entry && !_cachedParsers!.has(entry.id)) needed.add(entry);
+  }
+  for (const entry of needed) {
+    await loadLanguage(entry);
+  }
+  return _cachedParsers!;
+}
+
+/**
+ * Load ALL WASM grammars. Used by full builds and feature modules (CFG, dataflow, complexity)
+ * that may process files of any language.
+ */
+export async function createParsers(): Promise<Map<string, Parser | null>> {
+  if (_cachedParsers && _allParsersLoaded) return _cachedParsers;
+
+  await initParserRuntime();
   for (const entry of LANGUAGE_REGISTRY) {
-    try {
-      const lang = await Language.load(grammarPath(entry.grammarFile));
-      const parser = new Parser();
-      parser.setLanguage(lang);
-      parsers.set(entry.id, parser);
-      languages.set(entry.id, lang);
-      // Compile and cache tree-sitter Query for JS/TS/TSX extractors
-      if (entry.extractor === extractSymbols && !_queryCache.has(entry.id)) {
-        const isTS = entry.id === 'typescript' || entry.id === 'tsx';
-        const patterns = isTS
-          ? [...COMMON_QUERY_PATTERNS, ...TS_EXTRA_PATTERNS]
-          : [...COMMON_QUERY_PATTERNS, JS_CLASS_PATTERN];
-        _queryCache.set(entry.id, new Query(lang, patterns.join('\n')));
-      }
-    } catch (e: unknown) {
-      if (entry.required) throw e;
-      warn(
-        `${entry.id} parser failed to initialize: ${(e as Error).message}. ${entry.id} files will be skipped.`,
-      );
-      parsers.set(entry.id, null);
+    if (!_cachedParsers!.has(entry.id)) {
+      await loadLanguage(entry);
     }
   }
-  _cachedParsers = parsers;
-  _cachedLanguages = languages;
-  return parsers;
+  _allParsersLoaded = true;
+  return _cachedParsers!;
 }
 
 /**
@@ -193,42 +244,32 @@ export async function createParsers(): Promise<Map<string, Parser | null>> {
  * Call this between repeated builds in the same process (e.g. benchmarks)
  * to prevent memory accumulation that can cause segfaults.
  */
-export function disposeParsers(): void {
-  if (_cachedParsers) {
-    for (const [id, parser] of _cachedParsers) {
-      if (parser && typeof parser.delete === 'function') {
-        try {
-          parser.delete();
-        } catch (e: unknown) {
-          debug(`Failed to dispose parser ${id}: ${(e as Error).message}`);
-        }
-      }
-    }
-    _cachedParsers = null;
-  }
-  for (const [id, query] of _queryCache) {
-    if (query && typeof query.delete === 'function') {
+function disposeMapEntries(entries: Iterable<[string, any]>, label: string): void {
+  for (const [id, item] of entries) {
+    if (item && typeof item.delete === 'function') {
       try {
-        query.delete();
+        item.delete();
       } catch (e: unknown) {
-        debug(`Failed to dispose query ${id}: ${(e as Error).message}`);
+        debug(`Failed to dispose ${label} ${id}: ${(e as Error).message}`);
       }
     }
   }
+}
+
+export function disposeParsers(): void {
+  if (_cachedParsers) {
+    disposeMapEntries(_cachedParsers, 'parser');
+    _cachedParsers = null;
+  }
+  disposeMapEntries(_queryCache, 'query');
   _queryCache.clear();
   if (_cachedLanguages) {
-    for (const [id, lang] of _cachedLanguages) {
-      if (lang && typeof (lang as any).delete === 'function') {
-        try {
-          (lang as any).delete();
-        } catch (e: unknown) {
-          debug(`Failed to dispose language ${id}: ${(e as Error).message}`);
-        }
-      }
-    }
+    disposeMapEntries(_cachedLanguages, 'language');
     _cachedLanguages = null;
   }
   _initialized = false;
+  _allParsersLoaded = false;
+  _loadingPromises.clear();
 }
 
 export function getParser(parsers: Map<string, Parser | null>, filePath: string): Parser | null {
@@ -247,20 +288,15 @@ export async function ensureWasmTrees(
   fileSymbols: Map<string, any>,
   rootDir: string,
 ): Promise<void> {
-  // Check if any file needs a tree
-  let needsParse = false;
+  // Single pass: collect absolute paths for files that need parsing
+  const filePaths: string[] = [];
   for (const [relPath, symbols] of fileSymbols) {
-    if (!symbols._tree) {
-      const ext = path.extname(relPath).toLowerCase();
-      if (_extToLang.has(ext)) {
-        needsParse = true;
-        break;
-      }
+    if (!symbols._tree && _extToLang.has(path.extname(relPath).toLowerCase())) {
+      filePaths.push(path.join(rootDir, relPath));
     }
   }
-  if (!needsParse) return;
-
-  const parsers = await createParsers();
+  if (filePaths.length === 0) return;
+  const parsers = await ensureParsersForFiles(filePaths);
 
   for (const [relPath, symbols] of fileSymbols) {
     if (symbols._tree) continue;
@@ -349,17 +385,19 @@ function patchImports(imports: any[]): void {
   }
 }
 
-/** Normalize native typeMap array to a Map instance. */
+/** Normalize native typeMap array to a Map instance.
+ *  Uses first-wins semantics at equal confidence to match the WASM/JS extractor. */
 function patchTypeMap(r: any): void {
   if (!r.typeMap) {
     r.typeMap = new Map();
   } else if (!(r.typeMap instanceof Map)) {
-    r.typeMap = new Map(
-      r.typeMap.map((e: { name: string; typeName: string }) => [
-        e.name,
-        { type: e.typeName, confidence: 0.9 } as TypeMapEntry,
-      ]),
-    );
+    const map = new Map<string, TypeMapEntry>();
+    for (const e of r.typeMap as Array<{ name: string; typeName: string }>) {
+      if (!map.has(e.name)) {
+        map.set(e.name, { type: e.typeName, confidence: (e as any).confidence ?? 0.9 });
+      }
+    }
+    r.typeMap = map;
   }
 }
 
@@ -652,10 +690,17 @@ for (const entry of LANGUAGE_REGISTRY) {
 export const SUPPORTED_EXTENSIONS: Set<string> = new Set(_extToLang.keys());
 
 /**
- * WASM-based typeMap backfill for older native binaries that don't emit typeMap.
+ * WASM-based typeMap backfill for TS/TSX files parsed by the native engine.
+ * Serves two purposes:
+ * 1. Compatibility with older native binaries that don't emit typeMap (< 3.2.0).
+ * 2. Workaround for native parser scope-collision bugs — when the same variable
+ *    name appears at multiple scopes, native type extraction can produce
+ *    incorrect results. WASM's JS-based extractor handles scope traversal
+ *    more accurately. TODO: Remove purpose (2) once the Rust extractor handles
+ *    nested scopes correctly.
+ *
  * Uses tree-sitter AST extraction instead of regex to avoid false positives from
  * matches inside comments and string literals.
- * TODO: Remove once all published native binaries include typeMap extraction (>= 3.2.0)
  */
 async function backfillTypeMap(
   filePath: string,
@@ -670,7 +715,7 @@ async function backfillTypeMap(
       return { typeMap: new Map(), backfilled: false };
     }
   }
-  const parsers = await createParsers();
+  const parsers = await ensureParsersForFiles([filePath]);
   const extracted = wasmExtractSymbols(parsers, filePath, code);
   try {
     if (!extracted || extracted.symbols.typeMap.size === 0) {
@@ -732,23 +777,26 @@ export async function parseFileAuto(
     const result = native.parseFile(filePath, source, !!opts.dataflow, opts.ast !== false);
     if (!result) return null;
     const patched = patchNativeResult(result);
-    // Only backfill typeMap for TS/TSX — JS files have no type annotations,
-    // and the native engine already handles `new Expr()` patterns.
-    if (patched.typeMap.size === 0 && TS_BACKFILL_EXTS.has(path.extname(filePath))) {
+    // Always backfill typeMap for TS/TSX from WASM — native parser's type
+    // extraction can produce incorrect scope-collision results. Non-TS files
+    // are skipped to stay consistent with the batch path (backfillTypeMapBatch).
+    if (TS_BACKFILL_EXTS.has(path.extname(filePath))) {
       const { typeMap, backfilled } = await backfillTypeMap(filePath, source);
-      patched.typeMap = typeMap;
-      if (backfilled) patched._typeMapBackfilled = true;
+      if (backfilled) {
+        patched.typeMap = typeMap;
+        patched._typeMapBackfilled = true;
+      }
     }
     return patched;
   }
 
   // WASM path
-  const parsers = await createParsers();
+  const parsers = await ensureParsersForFiles([filePath]);
   const extracted = wasmExtractSymbols(parsers, filePath, source);
   return extracted ? extracted.symbols : null;
 }
 
-/** Backfill typeMap via WASM for files missing type-map data from native engine. */
+/** Backfill typeMap via WASM for TS/TSX files parsed by the native engine. */
 async function backfillTypeMapBatch(
   needsTypeMap: { filePath: string; relPath: string }[],
   result: Map<string, ExtractorOutput>,
@@ -758,7 +806,7 @@ async function backfillTypeMapBatch(
   );
   if (tsFiles.length === 0) return;
 
-  const parsers = await createParsers();
+  const parsers = await ensureParsersForFiles(tsFiles.map((f) => f.filePath));
   for (const { filePath, relPath } of tsFiles) {
     let extracted: WasmExtractResult | null | undefined;
     try {
@@ -790,7 +838,7 @@ async function parseFilesWasm(
   rootDir: string,
 ): Promise<Map<string, ExtractorOutput>> {
   const result = new Map<string, ExtractorOutput>();
-  const parsers = await createParsers();
+  const parsers = await ensureParsersForFiles(filePaths);
   for (const filePath of filePaths) {
     let code: string;
     try {
@@ -831,7 +879,12 @@ export async function parseFilesAuto(
     const patched = patchNativeResult(r);
     const relPath = path.relative(rootDir, r.file).split(path.sep).join('/');
     result.set(relPath, patched);
-    if (patched.typeMap.size === 0) {
+    // Always backfill TS/TSX type maps from WASM — the native parser's type
+    // extraction can produce incorrect results when the same variable name
+    // appears at multiple scopes (e.g. `node: TreeSitterNode` in one function
+    // vs `node: NodeRow` in another). The WASM JS extractor handles scope
+    // traversal order more accurately.
+    if (TS_BACKFILL_EXTS.has(path.extname(r.file))) {
       needsTypeMap.push({ filePath: r.file, relPath });
     }
   }
@@ -889,12 +942,13 @@ export async function parseFileIncremental(
     const result = cache.parseFile(filePath, source);
     if (!result) return null;
     const patched = patchNativeResult(result);
-    // Only backfill typeMap for TS/TSX — JS files have no type annotations,
-    // and the native engine already handles `new Expr()` patterns.
-    if (patched.typeMap.size === 0 && TS_BACKFILL_EXTS.has(path.extname(filePath))) {
+    // Always backfill typeMap for TS/TSX from WASM (see parseFileAuto comment).
+    if (TS_BACKFILL_EXTS.has(path.extname(filePath))) {
       const { typeMap, backfilled } = await backfillTypeMap(filePath, source);
-      patched.typeMap = typeMap;
-      if (backfilled) patched._typeMapBackfilled = true;
+      if (backfilled) {
+        patched.typeMap = typeMap;
+        patched._typeMapBackfilled = true;
+      }
     }
     return patched;
   }

package/src/domain/search/generator.ts

@@ -100,7 +100,7 @@ export async function buildEmbeddings(
   let overflowCount = 0;
 
   for (const [file, fileNodes] of byFile) {
-    const fullPath = path.join(rootDir, file);
+    const fullPath = path.isAbsolute(file) ? file : path.join(rootDir, file);
     let lines: string[];
     try {
       lines = fs.readFileSync(fullPath, 'utf-8').split('\n');

package/src/domain/search/models.ts

@@ -74,6 +74,7 @@ export const MODELS: Record<string, ModelConfig> = {
 export const EMBEDDING_STRATEGIES: readonly string[] = ['structured', 'source'];
 
 export const DEFAULT_MODEL: string = 'nomic-v1.5';
+const NPM_BIN = process.platform === 'win32' ? 'npm.cmd' : 'npm';
 const BATCH_SIZE_MAP: Record<string, number> = {
   minilm: 32,
   'jina-small': 16,
@@ -96,13 +97,28 @@ export function getModelConfig(modelKey?: string): ModelConfig {
 }
 
 /**
- * Prompt the user to install a missing package interactively.
+ * Attempt to install a missing package.
+ * In TTY environments, prompts the user for confirmation first.
+ * In non-TTY environments (CI, piped stdin), installs automatically with a log message.
  * Returns true if the package was installed, false otherwise.
- * Skips the prompt entirely in non-TTY environments (CI, piped stdin).
  * @internal Not part of the public barrel.
  */
 export function promptInstall(packageName: string): Promise<boolean> {
-  if (!process.stdin.isTTY) return Promise.resolve(false);
+  if (!process.stdin.isTTY) {
+    info(`Installing ${packageName} (optional dependency for semantic search)…`);
+    try {
+      execFileSync(NPM_BIN, ['install', '--no-save', packageName], {
+        stdio: 'inherit',
+        timeout: 300_000,
+      });
+      return Promise.resolve(true);
+    } catch (err) {
+      info(
+        `Auto-install of ${packageName} failed (${err instanceof Error ? err.message : String(err)}). Install it manually with:\n  npm install ${packageName}`,
+      );
+      return Promise.resolve(false);
+    }
+  }
 
   return new Promise((resolve) => {
     const rl = createInterface({ input: process.stdin, output: process.stderr });
@@ -112,12 +128,15 @@ export function promptInstall(packageName: string): Promise<boolean> {
         rl.close();
         if (answer.trim().toLowerCase() !== 'y') return resolve(false);
         try {
-          execFileSync('npm', ['install', packageName], {
+          execFileSync(NPM_BIN, ['install', packageName], {
             stdio: 'inherit',
             timeout: 300_000,
           });
           resolve(true);
-        } catch {
+        } catch (err) {
+          info(
+            `Install of ${packageName} failed (${err instanceof Error ? err.message : String(err)}). Install it manually with:\n  npm install ${packageName}`,
+          );
           resolve(false);
         }
       },
@@ -128,7 +147,7 @@ export function promptInstall(packageName: string): Promise<boolean> {
 /**
  * Lazy-load @huggingface/transformers.
  * If the package is missing, prompts the user to install it interactively.
- * In non-TTY environments, prints an error and exits.
+ * In non-TTY environments, attempts automatic installation.
  * @internal Not part of the public barrel.
  */
 export async function loadTransformers(): Promise<unknown> {
@@ -175,7 +194,7 @@ async function loadModel(modelKey?: string): Promise<{ extractor: unknown; confi
   pipeline = transformers.pipeline;
 
   info(`Loading embedding model: ${config.name} (${config.dim}d)...`);
-  const pipelineOpts = config.quantized ? { quantized: true } : {};
+  const pipelineOpts = config.quantized ? { dtype: 'q8' } : {};
   try {
     extractor =
       await // biome-ignore lint/complexity/noBannedTypes: dynamically loaded transformers pipeline is untyped

package/src/domain/search/search/hybrid.ts

@@ -25,45 +25,51 @@ export interface HybridSearchResult {
   results: HybridResult[];
 }
 
-export async function hybridSearchData(
-  query: string,
-  customDbPath: string | undefined,
-  opts: SemanticSearchOpts = {},
-): Promise<HybridSearchResult | null> {
-  const config = opts.config || loadConfig();
-  const searchCfg = config.search || ({} as CodegraphConfig['search']);
-  const limit = opts.limit ?? searchCfg.topK ?? 15;
-  const k = opts.rrfK ?? searchCfg.rrfK ?? 60;
-  const topK = (opts.limit ?? searchCfg.topK ?? 15) * 5;
-
-  const queries =
-    typeof query === 'string'
-      ? query
-          .split(';')
-          .map((q) => q.trim())
-          .filter((q) => q.length > 0)
-      : [query];
+interface RankedItem {
+  key: string;
+  rank: number;
+  source: 'bm25' | 'semantic';
+  name: string;
+  kind: string;
+  file: string;
+  line: number;
+  endLine?: number | null;
+  role?: string | null;
+  fileHash?: string | null;
+  bm25Score?: number;
+  similarity?: number;
+}
 
-  const checkDb = openReadonlyOrFail(customDbPath) as BetterSqlite3Database;
-  const ftsAvailable = hasFtsIndex(checkDb);
-  checkDb.close();
-  if (!ftsAvailable) return null;
+interface FusionEntry {
+  name: string;
+  kind: string;
+  file: string;
+  line: number;
+  endLine: number | null;
+  role: string | null;
+  fileHash: string | null;
+  rrfScore: number;
+  bm25Score: number | null;
+  bm25Rank: number | null;
+  similarity: number | null;
+  semanticRank: number | null;
+}
 
-  interface RankedItem {
-    key: string;
-    rank: number;
-    source: 'bm25' | 'semantic';
-    name: string;
-    kind: string;
-    file: string;
-    line: number;
-    endLine?: number | null;
-    role?: string | null;
-    fileHash?: string | null;
-    bm25Score?: number;
-    similarity?: number;
-  }
+/** Parse a semicolon-delimited query string into individual queries. */
+function parseQueries(query: string): string[] {
+  return query
+    .split(';')
+    .map((q) => q.trim())
+    .filter((q) => q.length > 0);
+}
 
+/** Collect BM25 and semantic ranked lists for each query. */
+async function collectRankedLists(
+  queries: string[],
+  customDbPath: string | undefined,
+  opts: SemanticSearchOpts,
+  topK: number,
+): Promise<RankedItem[][]> {
   const rankedLists: RankedItem[][] = [];
 
   for (const q of queries) {
@@ -96,22 +102,13 @@ export async function hybridSearchData(
     }
   }
 
-  interface FusionEntry {
-    name: string;
-    kind: string;
-    file: string;
-    line: number;
-    endLine: number | null;
-    role: string | null;
-    fileHash: string | null;
-    rrfScore: number;
-    bm25Score: number | null;
-    bm25Rank: number | null;
-    similarity: number | null;
-    semanticRank: number | null;
-  }
+  return rankedLists;
+}
 
+/** Reciprocal Rank Fusion: merge ranked lists into a single scored result set. */
+function fuseResults(rankedLists: RankedItem[][], k: number, limit: number): HybridResult[] {
   const fusionMap = new Map<string, FusionEntry>();
+
   for (const list of rankedLists) {
     for (const item of list) {
       if (!fusionMap.has(item.key)) {
@@ -146,7 +143,7 @@ export async function hybridSearchData(
     }
   }
 
-  const results: HybridResult[] = [...fusionMap.values()]
+  return [...fusionMap.values()]
     .sort((a, b) => b.rrfScore - a.rrfScore)
     .slice(0, limit)
     .map((e) => ({
@@ -163,6 +160,27 @@ export async function hybridSearchData(
       similarity: e.similarity,
       semanticRank: e.semanticRank,
     }));
+}
+
+export async function hybridSearchData(
+  query: string,
+  customDbPath: string | undefined,
+  opts: SemanticSearchOpts = {},
+): Promise<HybridSearchResult | null> {
+  const config = opts.config || loadConfig();
+  const searchCfg = config.search || ({} as CodegraphConfig['search']);
+  const limit = opts.limit ?? searchCfg.topK ?? 15;
+  const k = opts.rrfK ?? searchCfg.rrfK ?? 60;
+  const topK = (opts.limit ?? searchCfg.topK ?? 15) * 5;
+
+  const checkDb = openReadonlyOrFail(customDbPath) as BetterSqlite3Database;
+  const ftsAvailable = hasFtsIndex(checkDb);
+  checkDb.close();
+  if (!ftsAvailable) return null;
+
+  const queries = parseQueries(query);
+  const rankedLists = await collectRankedLists(queries, customDbPath, opts, topK);
+  const results = fuseResults(rankedLists, k, limit);
 
   return { results };
 }