gitnexus 1.1.9 → 1.2.1

package/dist/core/ingestion/filesystem-walker.js

@@ -2,6 +2,7 @@ import fs from 'fs/promises';
 import path from 'path';
 import { glob } from 'glob';
 import { shouldIgnorePath } from '../../config/ignore-service.js';
+const READ_CONCURRENCY = 32;
 export const walkRepository = async (repoPath, onProgress) => {
     const files = await glob('**/*', {
         cwd: repoPath,
@@ -10,16 +11,20 @@ export const walkRepository = async (repoPath, onProgress) => {
     });
     const filtered = files.filter(file => !shouldIgnorePath(file));
     const entries = [];
-    for (let i = 0; i < filtered.length; i++) {
-        const relativePath = filtered[i];
-        const fullPath = path.join(repoPath, relativePath);
-        try {
-            const content = await fs.readFile(fullPath, 'utf-8');
-            entries.push({ path: relativePath.replace(/\\/g, '/'), content });
-            onProgress?.(i + 1, filtered.length, relativePath);
-        }
-        catch {
-            onProgress?.(i + 1, filtered.length, relativePath);
+    let processed = 0;
+    for (let start = 0; start < filtered.length; start += READ_CONCURRENCY) {
+        const batch = filtered.slice(start, start + READ_CONCURRENCY);
+        const results = await Promise.allSettled(batch.map(relativePath => fs.readFile(path.join(repoPath, relativePath), 'utf-8')
+            .then(content => ({ path: relativePath.replace(/\\/g, '/'), content }))));
+        for (const result of results) {
+            processed++;
+            if (result.status === 'fulfilled') {
+                entries.push(result.value);
+                onProgress?.(processed, filtered.length, result.value.path);
+            }
+            else {
+                onProgress?.(processed, filtered.length, batch[results.indexOf(result)]);
+            }
         }
     }
     return entries;

package/dist/core/ingestion/heritage-processor.d.ts

@@ -8,7 +8,13 @@
 import { KnowledgeGraph } from '../graph/types.js';
 import { ASTCache } from './ast-cache.js';
 import { SymbolTable } from './symbol-table.js';
+import type { ExtractedHeritage } from './workers/parse-worker.js';
 export declare const processHeritage: (graph: KnowledgeGraph, files: {
     path: string;
     content: string;
 }[], astCache: ASTCache, symbolTable: SymbolTable, onProgress?: (current: number, total: number) => void) => Promise<void>;
+/**
+ * Fast path: resolve pre-extracted heritage from workers.
+ * No AST parsing — workers already extracted className + parentName + kind.
+ */
+export declare const processHeritageFromExtracted: (graph: KnowledgeGraph, extractedHeritage: ExtractedHeritage[], symbolTable: SymbolTable, onProgress?: (current: number, total: number) => void) => Promise<void>;

package/dist/core/ingestion/heritage-processor.js

@@ -39,6 +39,8 @@ export const processHeritage = async (graph, files, astCache, symbolTable, onPro
                 continue;
             }
             wasReparsed = true;
+            // Cache re-parsed tree for potential future use
+            astCache.set(file.path, tree);
         }
         let query;
         let matches;
@@ -49,8 +51,6 @@ export const processHeritage = async (graph, files, astCache, symbolTable, onPro
         }
         catch (queryError) {
             console.warn(`Heritage query error for ${file.path}:`, queryError);
-            if (wasReparsed)
-                tree.delete?.();
             continue;
         }
         // 4. Process heritage matches
@@ -126,9 +126,72 @@ export const processHeritage = async (graph, files, astCache, symbolTable, onPro
                 }
             }
         });
-        // Cleanup
-        if (wasReparsed) {
-            tree.delete?.();
+        // Tree is now owned by the LRU cache — no manual delete needed
+    }
+};
+/**
+ * Fast path: resolve pre-extracted heritage from workers.
+ * No AST parsing — workers already extracted className + parentName + kind.
+ */
+export const processHeritageFromExtracted = async (graph, extractedHeritage, symbolTable, onProgress) => {
+    const total = extractedHeritage.length;
+    for (let i = 0; i < extractedHeritage.length; i++) {
+        if (i % 500 === 0) {
+            onProgress?.(i, total);
+            await yieldToEventLoop();
+        }
+        const h = extractedHeritage[i];
+        if (h.kind === 'extends') {
+            const childId = symbolTable.lookupExact(h.filePath, h.className) ||
+                symbolTable.lookupFuzzy(h.className)[0]?.nodeId ||
+                generateId('Class', `${h.filePath}:${h.className}`);
+            const parentId = symbolTable.lookupFuzzy(h.parentName)[0]?.nodeId ||
+                generateId('Class', `${h.parentName}`);
+            if (childId && parentId && childId !== parentId) {
+                graph.addRelationship({
+                    id: generateId('EXTENDS', `${childId}->${parentId}`),
+                    sourceId: childId,
+                    targetId: parentId,
+                    type: 'EXTENDS',
+                    confidence: 1.0,
+                    reason: '',
+                });
+            }
+        }
+        else if (h.kind === 'implements') {
+            const classId = symbolTable.lookupExact(h.filePath, h.className) ||
+                symbolTable.lookupFuzzy(h.className)[0]?.nodeId ||
+                generateId('Class', `${h.filePath}:${h.className}`);
+            const interfaceId = symbolTable.lookupFuzzy(h.parentName)[0]?.nodeId ||
+                generateId('Interface', `${h.parentName}`);
+            if (classId && interfaceId) {
+                graph.addRelationship({
+                    id: generateId('IMPLEMENTS', `${classId}->${interfaceId}`),
+                    sourceId: classId,
+                    targetId: interfaceId,
+                    type: 'IMPLEMENTS',
+                    confidence: 1.0,
+                    reason: '',
+                });
+            }
+        }
+        else if (h.kind === 'trait-impl') {
+            const structId = symbolTable.lookupExact(h.filePath, h.className) ||
+                symbolTable.lookupFuzzy(h.className)[0]?.nodeId ||
+                generateId('Struct', `${h.filePath}:${h.className}`);
+            const traitId = symbolTable.lookupFuzzy(h.parentName)[0]?.nodeId ||
+                generateId('Trait', `${h.parentName}`);
+            if (structId && traitId) {
+                graph.addRelationship({
+                    id: generateId('IMPLEMENTS', `${structId}->${traitId}`),
+                    sourceId: structId,
+                    targetId: traitId,
+                    type: 'IMPLEMENTS',
+                    confidence: 1.0,
+                    reason: 'trait-impl',
+                });
+            }
         }
     }
+    onProgress?.(total, total);
 };

package/dist/core/ingestion/import-processor.d.ts

@@ -1,8 +1,30 @@
 import { KnowledgeGraph } from '../graph/types.js';
 import { ASTCache } from './ast-cache.js';
+import type { ExtractedImport } from './workers/parse-worker.js';
 export type ImportMap = Map<string, Set<string>>;
 export declare const createImportMap: () => ImportMap;
+/**
+ * Build a suffix index for O(1) endsWith lookups.
+ * Maps every possible path suffix to its original file path.
+ * e.g. for "src/com/example/Foo.java":
+ *   "Foo.java" -> "src/com/example/Foo.java"
+ *   "example/Foo.java" -> "src/com/example/Foo.java"
+ *   "com/example/Foo.java" -> "src/com/example/Foo.java"
+ *   etc.
+ */
+export interface SuffixIndex {
+    /** Exact suffix lookup (case-sensitive) */
+    get(suffix: string): string | undefined;
+    /** Case-insensitive suffix lookup */
+    getInsensitive(suffix: string): string | undefined;
+    /** Get all files in a directory suffix */
+    getFilesInDir(dirSuffix: string, extension: string): string[];
+}
 export declare const processImports: (graph: KnowledgeGraph, files: {
     path: string;
     content: string;
 }[], astCache: ASTCache, importMap: ImportMap, onProgress?: (current: number, total: number) => void, repoRoot?: string) => Promise<void>;
+export declare const processImportsFromExtracted: (graph: KnowledgeGraph, files: {
+    path: string;
+    content: string;
+}[], extractedImports: ExtractedImport[], importMap: ImportMap, onProgress?: (current: number, total: number) => void, repoRoot?: string) => Promise<void>;

package/dist/core/ingestion/import-processor.js

@@ -101,11 +101,73 @@ function tryResolveWithExtensions(basePath, allFiles) {
     }
     return null;
 }
+function buildSuffixIndex(normalizedFileList, allFileList) {
+    // Map: normalized suffix -> original file path
+    const exactMap = new Map();
+    // Map: lowercase suffix -> original file path
+    const lowerMap = new Map();
+    // Map: directory suffix -> list of file paths in that directory
+    const dirMap = new Map();
+    for (let i = 0; i < normalizedFileList.length; i++) {
+        const normalized = normalizedFileList[i];
+        const original = allFileList[i];
+        const parts = normalized.split('/');
+        // Index all suffixes: "a/b/c.java" -> ["c.java", "b/c.java", "a/b/c.java"]
+        for (let j = parts.length - 1; j >= 0; j--) {
+            const suffix = parts.slice(j).join('/');
+            // Only store first match (longest path wins for ambiguous suffixes)
+            if (!exactMap.has(suffix)) {
+                exactMap.set(suffix, original);
+            }
+            const lower = suffix.toLowerCase();
+            if (!lowerMap.has(lower)) {
+                lowerMap.set(lower, original);
+            }
+        }
+        // Index directory membership
+        const lastSlash = normalized.lastIndexOf('/');
+        if (lastSlash >= 0) {
+            // Build all directory suffixes
+            const dirParts = parts.slice(0, -1);
+            const fileName = parts[parts.length - 1];
+            const ext = fileName.substring(fileName.lastIndexOf('.'));
+            for (let j = dirParts.length - 1; j >= 0; j--) {
+                const dirSuffix = dirParts.slice(j).join('/');
+                const key = `${dirSuffix}:${ext}`;
+                let list = dirMap.get(key);
+                if (!list) {
+                    list = [];
+                    dirMap.set(key, list);
+                }
+                list.push(original);
+            }
+        }
+    }
+    return {
+        get: (suffix) => exactMap.get(suffix),
+        getInsensitive: (suffix) => lowerMap.get(suffix.toLowerCase()),
+        getFilesInDir: (dirSuffix, extension) => {
+            return dirMap.get(`${dirSuffix}:${extension}`) || [];
+        },
+    };
+}
 /**
- * Suffix-based resolution: try progressively shorter suffixes against all files.
- * Used for package-style imports (Java, Python, etc.).
+ * Suffix-based resolution using index. O(1) per lookup instead of O(files).
  */
-function suffixResolve(pathParts, normalizedFileList, allFileList) {
+function suffixResolve(pathParts, normalizedFileList, allFileList, index) {
+    if (index) {
+        for (let i = 0; i < pathParts.length; i++) {
+            const suffix = pathParts.slice(i).join('/');
+            for (const ext of EXTENSIONS) {
+                const suffixWithExt = suffix + ext;
+                const result = index.get(suffixWithExt) || index.getInsensitive(suffixWithExt);
+                if (result)
+                    return result;
+            }
+        }
+        return null;
+    }
+    // Fallback: linear scan (for backward compatibility)
     for (let i = 0; i < pathParts.length; i++) {
         const suffix = pathParts.slice(i).join('/');
         for (const ext of EXTENSIONS) {
@@ -129,7 +191,7 @@ function suffixResolve(pathParts, normalizedFileList, allFileList) {
  * Java wildcards and Go package imports are handled separately in processImports
  * because they resolve to multiple files.
  */
-const resolveImportPath = (currentFile, importPath, allFiles, allFileList, normalizedFileList, resolveCache, language, tsconfigPaths) => {
+const resolveImportPath = (currentFile, importPath, allFiles, allFileList, normalizedFileList, resolveCache, language, tsconfigPaths, index) => {
     const cacheKey = `${currentFile}::${importPath}`;
     if (resolveCache.has(cacheKey))
         return resolveCache.get(cacheKey) ?? null;
@@ -154,7 +216,7 @@ const resolveImportPath = (currentFile, importPath, allFiles, allFileList, norma
                     return cache(resolved);
                 // Try suffix matching as fallback
                 const parts = rewritten.split('/').filter(Boolean);
-                const suffixResult = suffixResolve(parts, normalizedFileList, allFileList);
+                const suffixResult = suffixResolve(parts, normalizedFileList, allFileList, index);
                 if (suffixResult)
                     return cache(suffixResult);
             }
@@ -194,7 +256,7 @@ const resolveImportPath = (currentFile, importPath, allFiles, allFileList, norma
         ? importPath
         : importPath.replace(/\./g, '/');
     const pathParts = pathLike.split('/').filter(Boolean);
-    const resolved = suffixResolve(pathParts, normalizedFileList, allFileList);
+    const resolved = suffixResolve(pathParts, normalizedFileList, allFileList, index);
     return cache(resolved);
 };
 // ============================================================================
@@ -276,15 +338,29 @@ function tryRustModulePath(modulePath, allFiles) {
  * Resolve a Java wildcard import (com.example.*) to all matching .java files.
  * Returns an array of file paths.
  */
-function resolveJavaWildcard(importPath, normalizedFileList, allFileList) {
+function resolveJavaWildcard(importPath, normalizedFileList, allFileList, index) {
     // "com.example.util.*" -> "com/example/util"
     const packagePath = importPath.slice(0, -2).replace(/\./g, '/');
+    if (index) {
+        // Use directory index: get all .java files in this package directory
+        const candidates = index.getFilesInDir(packagePath, '.java');
+        // Filter to only direct children (no subdirectories)
+        const packageSuffix = '/' + packagePath + '/';
+        return candidates.filter(f => {
+            const normalized = f.replace(/\\/g, '/');
+            const idx = normalized.indexOf(packageSuffix);
+            if (idx < 0)
+                return false;
+            const afterPkg = normalized.substring(idx + packageSuffix.length);
+            return !afterPkg.includes('/');
+        });
+    }
+    // Fallback: linear scan
     const packageSuffix = '/' + packagePath + '/';
     const matches = [];
     for (let i = 0; i < normalizedFileList.length; i++) {
         const normalized = normalizedFileList[i];
         if (normalized.includes(packageSuffix) && normalized.endsWith('.java')) {
-            // Ensure the file is directly in the package (not a subdirectory)
             const afterPackage = normalized.substring(normalized.indexOf(packageSuffix) + packageSuffix.length);
             if (!afterPackage.includes('/')) {
                 matches.push(allFileList[i]);
@@ -297,7 +373,7 @@ function resolveJavaWildcard(importPath, normalizedFileList, allFileList) {
  * Try to resolve a Java static import by stripping the member name.
  * "com.example.Constants.VALUE" -> resolve "com.example.Constants"
  */
-function resolveJavaStaticImport(importPath, normalizedFileList, allFileList) {
+function resolveJavaStaticImport(importPath, normalizedFileList, allFileList, index) {
     // Static imports look like: com.example.Constants.VALUE or com.example.Constants.*
     // The last segment is a member name (field/method) if it starts with lowercase or is ALL_CAPS
     const segments = importPath.split('.');
@@ -307,10 +383,15 @@ function resolveJavaStaticImport(importPath, normalizedFileList, allFileList) {
     // If last segment is a wildcard or ALL_CAPS constant or starts with lowercase, strip it
     if (lastSeg === '*' || /^[a-z]/.test(lastSeg) || /^[A-Z_]+$/.test(lastSeg)) {
         const classPath = segments.slice(0, -1).join('/');
-        const classSuffix = '/' + classPath + '.java';
+        const classSuffix = classPath + '.java';
+        if (index) {
+            return index.get(classSuffix) || index.getInsensitive(classSuffix) || null;
+        }
+        // Fallback: linear scan
+        const fullSuffix = '/' + classSuffix;
         for (let i = 0; i < normalizedFileList.length; i++) {
-            if (normalizedFileList[i].endsWith(classSuffix) ||
-                normalizedFileList[i].toLowerCase().endsWith(classSuffix.toLowerCase())) {
+            if (normalizedFileList[i].endsWith(fullSuffix) ||
+                normalizedFileList[i].toLowerCase().endsWith(fullSuffix.toLowerCase())) {
                 return allFileList[i];
             }
         }
@@ -356,6 +437,8 @@ export const processImports = async (graph, files, astCache, importMap, onProgre
     const allFileList = files.map(f => f.path);
     // Pre-compute normalized file list once (forward slashes)
     const normalizedFileList = allFileList.map(p => p.replace(/\\/g, '/'));
+    // Build suffix index for O(1) lookups
+    const index = buildSuffixIndex(normalizedFileList, allFileList);
     // Track import statistics
     let totalImportsFound = 0;
     let totalImportsResolved = 0;
@@ -407,6 +490,8 @@ export const processImports = async (graph, files, astCache, importMap, onProgre
                 continue;
             }
             wasReparsed = true;
+            // Cache re-parsed tree so call/heritage phases get hits
+            astCache.set(file.path, tree);
         }
         let query;
         let matches;
@@ -447,14 +532,14 @@ export const processImports = async (graph, files, astCache, importMap, onProgre
                 // ---- Java: handle wildcards and static imports specially ----
                 if (language === SupportedLanguages.Java) {
                     if (rawImportPath.endsWith('.*')) {
-                        const matchedFiles = resolveJavaWildcard(rawImportPath, normalizedFileList, allFileList);
+                        const matchedFiles = resolveJavaWildcard(rawImportPath, normalizedFileList, allFileList, index);
                         for (const matchedFile of matchedFiles) {
                             addImportEdge(file.path, matchedFile);
                         }
                         return; // skip single-file resolution
                     }
                     // Try static import resolution (strip member name)
-                    const staticResolved = resolveJavaStaticImport(rawImportPath, normalizedFileList, allFileList);
+                    const staticResolved = resolveJavaStaticImport(rawImportPath, normalizedFileList, allFileList, index);
                     if (staticResolved) {
                         addImportEdge(file.path, staticResolved);
                         return;
@@ -473,18 +558,128 @@ export const processImports = async (graph, files, astCache, importMap, onProgre
                     // Fall through if no files found (package might be external)
                 }
                 // ---- Standard single-file resolution ----
-                const resolvedPath = resolveImportPath(file.path, rawImportPath, allFilePaths, allFileList, normalizedFileList, resolveCache, language, tsconfigPaths);
+                const resolvedPath = resolveImportPath(file.path, rawImportPath, allFilePaths, allFileList, normalizedFileList, resolveCache, language, tsconfigPaths, index);
                 if (resolvedPath) {
                     addImportEdge(file.path, resolvedPath);
                 }
             }
         });
-        // If re-parsed just for this, delete the tree to save memory
-        if (wasReparsed) {
-            tree.delete?.();
-        }
+        // Tree is now owned by the LRU cache — no manual delete needed
     }
     if (isDev) {
         console.log(`📊 Import processing complete: ${totalImportsResolved}/${totalImportsFound} imports resolved to graph edges`);
     }
 };
+// ============================================================================
+// FAST PATH: Resolve pre-extracted imports (no parsing needed)
+// ============================================================================
+export const processImportsFromExtracted = async (graph, files, extractedImports, importMap, onProgress, repoRoot) => {
+    const allFilePaths = new Set(files.map(f => f.path));
+    const resolveCache = new Map();
+    const allFileList = files.map(f => f.path);
+    const normalizedFileList = allFileList.map(p => p.replace(/\\/g, '/'));
+    // Build suffix index for O(1) lookups
+    const index = buildSuffixIndex(normalizedFileList, allFileList);
+    let totalImportsFound = 0;
+    let totalImportsResolved = 0;
+    const effectiveRoot = repoRoot || '';
+    const tsconfigPaths = await loadTsconfigPaths(effectiveRoot);
+    const goModule = await loadGoModulePath(effectiveRoot);
+    const addImportEdge = (filePath, resolvedPath) => {
+        const sourceId = generateId('File', filePath);
+        const targetId = generateId('File', resolvedPath);
+        const relId = generateId('IMPORTS', `${filePath}->${resolvedPath}`);
+        totalImportsResolved++;
+        graph.addRelationship({
+            id: relId,
+            sourceId,
+            targetId,
+            type: 'IMPORTS',
+            confidence: 1.0,
+            reason: '',
+        });
+        if (!importMap.has(filePath)) {
+            importMap.set(filePath, new Set());
+        }
+        importMap.get(filePath).add(resolvedPath);
+    };
+    // Group by file for progress reporting (users see file count, not import count)
+    const importsByFile = new Map();
+    for (const imp of extractedImports) {
+        let list = importsByFile.get(imp.filePath);
+        if (!list) {
+            list = [];
+            importsByFile.set(imp.filePath, list);
+        }
+        list.push(imp);
+    }
+    const totalFiles = importsByFile.size;
+    let filesProcessed = 0;
+    // Pre-build a suffix index for O(1) suffix lookups instead of O(n) linear scans
+    const suffixIndex = new Map();
+    for (let i = 0; i < normalizedFileList.length; i++) {
+        const normalized = normalizedFileList[i];
+        // Index by last path segment (filename) for fast suffix matching
+        const lastSlash = normalized.lastIndexOf('/');
+        const filename = lastSlash >= 0 ? normalized.substring(lastSlash + 1) : normalized;
+        let list = suffixIndex.get(filename);
+        if (!list) {
+            list = [];
+            suffixIndex.set(filename, list);
+        }
+        list.push(allFileList[i]);
+    }
+    for (const [filePath, fileImports] of importsByFile) {
+        filesProcessed++;
+        if (filesProcessed % 100 === 0) {
+            onProgress?.(filesProcessed, totalFiles);
+            await yieldToEventLoop();
+        }
+        for (const { rawImportPath, language } of fileImports) {
+            totalImportsFound++;
+            // Check resolve cache first
+            const cacheKey = `${filePath}::${rawImportPath}`;
+            if (resolveCache.has(cacheKey)) {
+                const cached = resolveCache.get(cacheKey);
+                if (cached)
+                    addImportEdge(filePath, cached);
+                continue;
+            }
+            // Java: handle wildcards and static imports
+            if (language === SupportedLanguages.Java) {
+                if (rawImportPath.endsWith('.*')) {
+                    const matchedFiles = resolveJavaWildcard(rawImportPath, normalizedFileList, allFileList, index);
+                    for (const matchedFile of matchedFiles) {
+                        addImportEdge(filePath, matchedFile);
+                    }
+                    continue;
+                }
+                const staticResolved = resolveJavaStaticImport(rawImportPath, normalizedFileList, allFileList, index);
+                if (staticResolved) {
+                    resolveCache.set(cacheKey, staticResolved);
+                    addImportEdge(filePath, staticResolved);
+                    continue;
+                }
+            }
+            // Go: handle package-level imports
+            if (language === SupportedLanguages.Go && goModule && rawImportPath.startsWith(goModule.modulePath)) {
+                const pkgFiles = resolveGoPackage(rawImportPath, goModule, normalizedFileList, allFileList);
+                if (pkgFiles.length > 0) {
+                    for (const pkgFile of pkgFiles) {
+                        addImportEdge(filePath, pkgFile);
+                    }
+                    continue;
+                }
+            }
+            // Standard resolution (has its own internal cache)
+            const resolvedPath = resolveImportPath(filePath, rawImportPath, allFilePaths, allFileList, normalizedFileList, resolveCache, language, tsconfigPaths, index);
+            if (resolvedPath) {
+                addImportEdge(filePath, resolvedPath);
+            }
+        }
+    }
+    onProgress?.(totalFiles, totalFiles);
+    if (isDev) {
+        console.log(`📊 Import processing (fast path): ${totalImportsResolved}/${totalImportsFound} imports resolved to graph edges`);
+    }
+};

package/dist/core/ingestion/parsing-processor.d.ts

@@ -1,8 +1,15 @@
 import { KnowledgeGraph } from '../graph/types.js';
 import { SymbolTable } from './symbol-table.js';
 import { ASTCache } from './ast-cache.js';
+import { WorkerPool } from './workers/worker-pool.js';
+import type { ExtractedImport, ExtractedCall, ExtractedHeritage } from './workers/parse-worker.js';
 export type FileProgressCallback = (current: number, total: number, filePath: string) => void;
+export interface WorkerExtractedData {
+    imports: ExtractedImport[];
+    calls: ExtractedCall[];
+    heritage: ExtractedHeritage[];
+}
 export declare const processParsing: (graph: KnowledgeGraph, files: {
     path: string;
     content: string;
-}[], symbolTable: SymbolTable, astCache: ASTCache, onFileProgress?: FileProgressCallback) => Promise<void>;
+}[], symbolTable: SymbolTable, astCache: ASTCache, onFileProgress?: FileProgressCallback, workerPool?: WorkerPool) => Promise<WorkerExtractedData | null>;