gitnexus 1.1.8 → 1.2.0

package/dist/core/ingestion/pipeline.js

@@ -1,20 +1,22 @@
 import { createKnowledgeGraph } from '../graph/graph.js';
 import { processStructure } from './structure-processor.js';
 import { processParsing } from './parsing-processor.js';
-import { processImports, createImportMap } from './import-processor.js';
-import { processCalls } from './call-processor.js';
-import { processHeritage } from './heritage-processor.js';
+import { processImports, processImportsFromExtracted, createImportMap } from './import-processor.js';
+import { processCalls, processCallsFromExtracted } from './call-processor.js';
+import { processHeritage, processHeritageFromExtracted } from './heritage-processor.js';
 import { processCommunities } from './community-processor.js';
 import { processProcesses } from './process-processor.js';
 import { createSymbolTable } from './symbol-table.js';
 import { createASTCache } from './ast-cache.js';
 import { walkRepository } from './filesystem-walker.js';
-const isDev = process.env.NODE_ENV !== 'production';
+import { createWorkerPool } from './workers/worker-pool.js';
+const isDev = process.env.NODE_ENV === 'development';
 export const runPipelineFromRepo = async (repoPath, onProgress) => {
     const graph = createKnowledgeGraph();
     const fileContents = new Map();
     const symbolTable = createSymbolTable();
-    const astCache = createASTCache(50);
+    // AST cache sized after file scan — start with a placeholder, resize after we know file count
+    let astCache = createASTCache(50);
     const importMap = createImportMap();
     const cleanup = () => {
         astCache.clear();
@@ -37,6 +39,8 @@ export const runPipelineFromRepo = async (repoPath, onProgress) => {
             });
         });
         files.forEach(f => fileContents.set(f.path, f.content));
+        // Resize AST cache to fit all files — avoids re-parsing in import/call/heritage phases
+        astCache = createASTCache(files.length);
         onProgress({
             phase: 'extracting',
             percent: 15,
@@ -63,31 +67,61 @@ export const runPipelineFromRepo = async (repoPath, onProgress) => {
             message: 'Parsing code definitions...',
             stats: { filesProcessed: 0, totalFiles: files.length, nodesCreated: graph.nodeCount },
         });
-        await processParsing(graph, files, symbolTable, astCache, (current, total, filePath) => {
-            const parsingProgress = 30 + ((current / total) * 40);
-            onProgress({
-                phase: 'parsing',
-                percent: Math.round(parsingProgress),
-                message: 'Parsing code definitions...',
-                detail: filePath,
-                stats: { filesProcessed: current, totalFiles: total, nodesCreated: graph.nodeCount },
-            });
-        });
+        // Create worker pool for parallel parsing, with graceful fallback
+        let workerPool;
+        try {
+            const workerUrl = new URL('./workers/parse-worker.js', import.meta.url);
+            workerPool = createWorkerPool(workerUrl);
+        }
+        catch (err) {
+            // Worker pool creation failed (e.g., single core) — sequential fallback
+        }
+        let workerData = null;
+        try {
+            workerData = await processParsing(graph, files, symbolTable, astCache, (current, total, filePath) => {
+                const parsingProgress = 30 + ((current / total) * 40);
+                onProgress({
+                    phase: 'parsing',
+                    percent: Math.round(parsingProgress),
+                    message: 'Parsing code definitions...',
+                    detail: filePath,
+                    stats: { filesProcessed: current, totalFiles: total, nodesCreated: graph.nodeCount },
+                });
+            }, workerPool);
+        }
+        finally {
+            await workerPool?.terminate();
+        }
         onProgress({
             phase: 'imports',
             percent: 70,
             message: 'Resolving imports...',
             stats: { filesProcessed: 0, totalFiles: files.length, nodesCreated: graph.nodeCount },
         });
-        await processImports(graph, files, astCache, importMap, (current, total) => {
-            const importProgress = 70 + ((current / total) * 12);
-            onProgress({
-                phase: 'imports',
-                percent: Math.round(importProgress),
-                message: 'Resolving imports...',
-                stats: { filesProcessed: current, totalFiles: total, nodesCreated: graph.nodeCount },
-            });
-        }, repoPath);
+        if (workerData) {
+            // Fast path: imports already extracted by workers, just resolve paths
+            await processImportsFromExtracted(graph, files, workerData.imports, importMap, (current, total) => {
+                const importProgress = 70 + ((current / total) * 12);
+                onProgress({
+                    phase: 'imports',
+                    percent: Math.round(importProgress),
+                    message: 'Resolving imports...',
+                    stats: { filesProcessed: current, totalFiles: total, nodesCreated: graph.nodeCount },
+                });
+            }, repoPath);
+        }
+        else {
+            // Fallback: full parse + resolve (sequential path)
+            await processImports(graph, files, astCache, importMap, (current, total) => {
+                const importProgress = 70 + ((current / total) * 12);
+                onProgress({
+                    phase: 'imports',
+                    percent: Math.round(importProgress),
+                    message: 'Resolving imports...',
+                    stats: { filesProcessed: current, totalFiles: total, nodesCreated: graph.nodeCount },
+                });
+            }, repoPath);
+        }
         if (isDev) {
             const importsCount = graph.relationships.filter(r => r.type === 'IMPORTS').length;
             console.log(`📊 Pipeline: After import phase, graph has ${importsCount} IMPORTS relationships (total: ${graph.relationshipCount})`);
@@ -98,30 +132,60 @@ export const runPipelineFromRepo = async (repoPath, onProgress) => {
             message: 'Tracing function calls...',
             stats: { filesProcessed: 0, totalFiles: files.length, nodesCreated: graph.nodeCount },
         });
-        await processCalls(graph, files, astCache, symbolTable, importMap, (current, total) => {
-            const callProgress = 82 + ((current / total) * 10);
-            onProgress({
-                phase: 'calls',
-                percent: Math.round(callProgress),
-                message: 'Tracing function calls...',
-                stats: { filesProcessed: current, totalFiles: total, nodesCreated: graph.nodeCount },
+        if (workerData) {
+            // Fast path: calls already extracted by workers, just resolve targets
+            await processCallsFromExtracted(graph, workerData.calls, symbolTable, importMap, (current, total) => {
+                const callProgress = 82 + ((current / total) * 10);
+                onProgress({
+                    phase: 'calls',
+                    percent: Math.round(callProgress),
+                    message: 'Tracing function calls...',
+                    stats: { filesProcessed: current, totalFiles: total, nodesCreated: graph.nodeCount },
+                });
             });
-        });
+        }
+        else {
+            // Fallback: full parse + resolve (sequential path)
+            await processCalls(graph, files, astCache, symbolTable, importMap, (current, total) => {
+                const callProgress = 82 + ((current / total) * 10);
+                onProgress({
+                    phase: 'calls',
+                    percent: Math.round(callProgress),
+                    message: 'Tracing function calls...',
+                    stats: { filesProcessed: current, totalFiles: total, nodesCreated: graph.nodeCount },
+                });
+            });
+        }
         onProgress({
             phase: 'heritage',
             percent: 92,
             message: 'Extracting class inheritance...',
             stats: { filesProcessed: 0, totalFiles: files.length, nodesCreated: graph.nodeCount },
         });
-        await processHeritage(graph, files, astCache, symbolTable, (current, total) => {
-            const heritageProgress = 88 + ((current / total) * 4);
-            onProgress({
-                phase: 'heritage',
-                percent: Math.round(heritageProgress),
-                message: 'Extracting class inheritance...',
-                stats: { filesProcessed: current, totalFiles: total, nodesCreated: graph.nodeCount },
+        if (workerData) {
+            // Fast path: heritage already extracted by workers, just resolve symbols
+            await processHeritageFromExtracted(graph, workerData.heritage, symbolTable, (current, total) => {
+                const heritageProgress = 88 + ((current / total) * 4);
+                onProgress({
+                    phase: 'heritage',
+                    percent: Math.round(heritageProgress),
+                    message: 'Extracting class inheritance...',
+                    stats: { filesProcessed: current, totalFiles: total, nodesCreated: graph.nodeCount },
+                });
             });
-        });
+        }
+        else {
+            // Fallback: full parse + resolve (sequential path)
+            await processHeritage(graph, files, astCache, symbolTable, (current, total) => {
+                const heritageProgress = 88 + ((current / total) * 4);
+                onProgress({
+                    phase: 'heritage',
+                    percent: Math.round(heritageProgress),
+                    message: 'Extracting class inheritance...',
+                    stats: { filesProcessed: current, totalFiles: total, nodesCreated: graph.nodeCount },
+                });
+            });
+        }
         onProgress({
             phase: 'communities',
             percent: 92,

package/dist/core/ingestion/process-processor.js

@@ -10,7 +10,7 @@
  * Processes help agents understand how features work through the codebase.
  */
 import { calculateEntryPointScore, isTestFile } from './entry-point-scoring.js';
-const isDev = process.env.NODE_ENV !== 'production';
+const isDev = process.env.NODE_ENV === 'development';
 const DEFAULT_CONFIG = {
     maxTraceDepth: 10,
     maxBranching: 4,

package/dist/core/ingestion/workers/parse-worker.d.ts

@@ -0,0 +1,58 @@
+interface ParsedNode {
+    id: string;
+    label: string;
+    properties: {
+        name: string;
+        filePath: string;
+        startLine: number;
+        endLine: number;
+        language: string;
+        isExported: boolean;
+    };
+}
+interface ParsedRelationship {
+    id: string;
+    sourceId: string;
+    targetId: string;
+    type: 'DEFINES';
+    confidence: number;
+    reason: string;
+}
+interface ParsedSymbol {
+    filePath: string;
+    name: string;
+    nodeId: string;
+    type: string;
+}
+export interface ExtractedImport {
+    filePath: string;
+    rawImportPath: string;
+    language: string;
+}
+export interface ExtractedCall {
+    filePath: string;
+    calledName: string;
+    /** generateId of enclosing function, or generateId('File', filePath) for top-level */
+    sourceId: string;
+}
+export interface ExtractedHeritage {
+    filePath: string;
+    className: string;
+    parentName: string;
+    /** 'extends' | 'implements' | 'trait-impl' */
+    kind: string;
+}
+export interface ParseWorkerResult {
+    nodes: ParsedNode[];
+    relationships: ParsedRelationship[];
+    symbols: ParsedSymbol[];
+    imports: ExtractedImport[];
+    calls: ExtractedCall[];
+    heritage: ExtractedHeritage[];
+    fileCount: number;
+}
+export interface ParseWorkerInput {
+    path: string;
+    content: string;
+}
+export {};