@zuvia-software-solutions/code-mapper 1.4.0

package/dist/core/ingestion/pipeline.js

@@ -0,0 +1,416 @@
+// code-mapper/src/core/ingestion/pipeline.ts
+/** @file pipeline.ts @description Main ingestion pipeline that orchestrates scanning, parsing, resolution, community detection, and process detection across chunked file batches */
+import { createKnowledgeGraph } from '../graph/graph.js';
+import { processStructure } from './structure-processor.js';
+import { processParsing } from './parsing-processor.js';
+import { processImports, processImportsFromExtracted, buildImportResolutionContext } from './import-processor.js';
+import { processCalls, processCallsFromExtracted, processRoutesFromExtracted, createDependsOnEdges, createProvidesEdges } from './call-processor.js';
+import { processHeritage, processHeritageFromExtracted } from './heritage-processor.js';
+import { computeMRO } from './mro-processor.js';
+import { processCommunities } from './community-processor.js';
+import { processProcesses } from './process-processor.js';
+import { createResolutionContext } from './resolution-context.js';
+import { createASTCache } from './ast-cache.js';
+import { walkRepositoryPaths, readFileContents } from './filesystem-walker.js';
+import { getLanguageFromFilename } from './utils.js';
+import { isLanguageAvailable } from '../tree-sitter/parser-loader.js';
+import { createWorkerPool } from './workers/worker-pool.js';
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+import { memoryGuard } from '../../lib/memory-guard.js';
+const isDev = process.env.NODE_ENV === 'development';
+// Default chunk budget — used when memory is plentiful.
+// Under memory pressure, adaptiveBatchSize() shrinks this automatically.
+const DEFAULT_CHUNK_BYTE_BUDGET = 50 * 1024 * 1024;
+// Estimated peak working memory multiplier per byte of source
+// (ASTs, symbol table entries, extracted data per chunk)
+const WORKING_MEMORY_MULTIPLIER = 20;
+// Max AST trees to keep in LRU cache
+const AST_CACHE_CAP = 50;
+export const runPipelineFromRepo = async (repoPath, onProgress) => {
+    const graph = createKnowledgeGraph();
+    const ctx = createResolutionContext();
+    const symbolTable = ctx.symbols;
+    let astCache = createASTCache(AST_CACHE_CAP);
+    const cleanup = () => {
+        astCache.clear();
+        ctx.clear();
+    };
+    try {
+        // Phase 1: Scan paths only (no content read)
+        onProgress({
+            phase: 'extracting',
+            percent: 0,
+            message: 'Scanning repository...',
+        });
+        const scannedFiles = await walkRepositoryPaths(repoPath, (current, total, filePath) => {
+            const scanProgress = Math.round((current / total) * 15);
+            onProgress({
+                phase: 'extracting',
+                percent: scanProgress,
+                message: 'Scanning repository...',
+                detail: filePath,
+                stats: { filesProcessed: current, totalFiles: total, nodesCreated: graph.nodeCount },
+            });
+        });
+        const totalFiles = scannedFiles.length;
+        onProgress({
+            phase: 'extracting',
+            percent: 15,
+            message: 'Repository scanned successfully',
+            stats: { filesProcessed: totalFiles, totalFiles, nodesCreated: graph.nodeCount },
+        });
+        // Phase 2: Structure (paths only, no content needed)
+        onProgress({
+            phase: 'structure',
+            percent: 15,
+            message: 'Analyzing project structure...',
+            stats: { filesProcessed: 0, totalFiles, nodesCreated: graph.nodeCount },
+        });
+        const allPaths = scannedFiles.map(f => f.path);
+        processStructure(graph, allPaths);
+        onProgress({
+            phase: 'structure',
+            percent: 20,
+            message: 'Project structure analyzed',
+            stats: { filesProcessed: totalFiles, totalFiles, nodesCreated: graph.nodeCount },
+        });
+        // Phase 3+4: Chunked read + parse
+        // Group parseable files into byte-budget chunks (~20MB at a time)
+        // Each chunk is: read -> parse -> extract -> free
+        const parseableScanned = scannedFiles.filter(f => {
+            const lang = getLanguageFromFilename(f.path);
+            return lang && isLanguageAvailable(lang);
+        });
+        // Log files skipped due to unavailable parsers
+        const skippedByLang = new Map();
+        for (const f of scannedFiles) {
+            const lang = getLanguageFromFilename(f.path);
+            if (lang && !isLanguageAvailable(lang)) {
+                skippedByLang.set(lang, (skippedByLang.get(lang) || 0) + 1);
+            }
+        }
+        for (const [lang, count] of skippedByLang) {
+            console.warn(`Skipping ${count} ${lang} file(s) — ${lang} parser not available (native binding may not have built). Try: npm rebuild tree-sitter-${lang}`);
+        }
+        const totalParseable = parseableScanned.length;
+        if (totalParseable === 0) {
+            onProgress({
+                phase: 'parsing',
+                percent: 82,
+                message: 'No parseable files found — skipping parsing phase',
+                stats: { filesProcessed: 0, totalFiles: 0, nodesCreated: graph.nodeCount },
+            });
+        }
+        // Build chunks based on adaptive byte budget — shrinks under memory pressure
+        const chunkBudget = memoryGuard.adaptiveBatchSize(WORKING_MEMORY_MULTIPLIER, DEFAULT_CHUNK_BYTE_BUDGET);
+        const chunks = [];
+        let currentChunk = [];
+        let currentBytes = 0;
+        for (const file of parseableScanned) {
+            if (currentChunk.length > 0 && currentBytes + file.size > chunkBudget) {
+                chunks.push(currentChunk);
+                currentChunk = [];
+                currentBytes = 0;
+            }
+            currentChunk.push(file.path);
+            currentBytes += file.size;
+        }
+        if (currentChunk.length > 0)
+            chunks.push(currentChunk);
+        const numChunks = chunks.length;
+        if (isDev) {
+            const totalMB = parseableScanned.reduce((s, f) => s + f.size, 0) / (1024 * 1024);
+            console.log(`📂 Scan: ${totalFiles} paths, ${totalParseable} parseable (${totalMB.toFixed(0)}MB), ${numChunks} chunks @ ${Math.round(chunkBudget / (1024 * 1024))}MB budget (${memoryGuard.summary()})`);
+        }
+        onProgress({
+            phase: 'parsing',
+            percent: 20,
+            message: `Parsing ${totalParseable} files in ${numChunks} chunk${numChunks !== 1 ? 's' : ''}...`,
+            stats: { filesProcessed: 0, totalFiles: totalParseable, nodesCreated: graph.nodeCount },
+        });
+        // Create worker pool once, reused across chunks
+        let workerPool;
+        try {
+            let workerUrl = new URL('./workers/parse-worker.js', import.meta.url);
+            // Under vitest, import.meta.url points to src/ where no .js exists;
+            // fall back to compiled dist/ worker for real worker threads
+            const thisDir = fileURLToPath(new URL('.', import.meta.url));
+            if (!fs.existsSync(fileURLToPath(workerUrl))) {
+                const distWorker = path.resolve(thisDir, '..', '..', '..', 'dist', 'core', 'ingestion', 'workers', 'parse-worker.js');
+                if (fs.existsSync(distWorker)) {
+                    workerUrl = pathToFileURL(distWorker);
+                }
+            }
+            workerPool = createWorkerPool(workerUrl);
+        }
+        catch (err) {
+            if (isDev)
+                console.warn('Worker pool creation failed, using sequential fallback:', err.message);
+        }
+        let filesParsedSoFar = 0;
+        // AST cache sized for one chunk (used by sequential fallback for import/call/heritage)
+        const maxChunkFiles = chunks.reduce((max, c) => Math.max(max, c.length), 0);
+        astCache = createASTCache(maxChunkFiles);
+        // Build import resolution context once; reused across all chunks to avoid
+        // rebuilding O(files * path_depth) structures
+        const importCtx = buildImportResolutionContext(allPaths);
+        const allPathObjects = allPaths.map(p => ({ path: p }));
+        // Two-phase: parse all chunks first (populating symbol table), then resolve calls
+        // Phase A: parse + resolve imports/heritage per chunk, COLLECT calls for deferred resolution
+        // Phase B: resolve ALL collected calls after all symbols are registered (fixes cross-chunk calls)
+        const sequentialChunkPaths = [];
+        // Deferred call data — collected across chunks, resolved after all parsing completes
+        const allExtractedCalls = [];
+        const allConstructorBindings = [];
+        try {
+            for (let chunkIdx = 0; chunkIdx < numChunks; chunkIdx++) {
+                const chunkPaths = chunks[chunkIdx];
+                // Read content for this chunk
+                const chunkContents = await readFileContents(repoPath, chunkPaths);
+                const chunkFiles = chunkPaths
+                    .filter(p => chunkContents.has(p))
+                    .map(p => ({ path: p, content: chunkContents.get(p) }));
+                // Parse chunk (workers or sequential fallback)
+                const chunkWorkerData = await processParsing(graph, chunkFiles, symbolTable, astCache, (current, _total, filePath) => {
+                    const globalCurrent = filesParsedSoFar + current;
+                    const parsingProgress = 20 + ((globalCurrent / totalParseable) * 62);
+                    onProgress({
+                        phase: 'parsing',
+                        percent: Math.round(parsingProgress),
+                        message: `Parsing chunk ${chunkIdx + 1}/${numChunks}...`,
+                        detail: filePath,
+                        stats: { filesProcessed: globalCurrent, totalFiles: totalParseable, nodesCreated: graph.nodeCount },
+                    });
+                }, workerPool);
+                const chunkBasePercent = 20 + ((filesParsedSoFar / totalParseable) * 62);
+                if (chunkWorkerData) {
+                    // Resolve imports per-chunk (file-level, doesn't need full symbol table)
+                    await processImportsFromExtracted(graph, allPathObjects, chunkWorkerData.imports, ctx, (current, total) => {
+                        onProgress({
+                            phase: 'parsing',
+                            percent: Math.round(chunkBasePercent),
+                            message: `Resolving imports (chunk ${chunkIdx + 1}/${numChunks})...`,
+                            detail: `${current}/${total} files`,
+                            stats: { filesProcessed: filesParsedSoFar, totalFiles: totalParseable, nodesCreated: graph.nodeCount },
+                        });
+                    }, repoPath, importCtx);
+                    // COLLECT calls for deferred resolution (don't resolve yet — callee may be in later chunk)
+                    allExtractedCalls.push(...chunkWorkerData.calls);
+                    if (chunkWorkerData.constructorBindings) {
+                        allConstructorBindings.push(...chunkWorkerData.constructorBindings);
+                    }
+                    // Heritage + Routes can resolve per-chunk (class-level, usually same-file)
+                    await Promise.all([
+                        processHeritageFromExtracted(graph, chunkWorkerData.heritage, ctx, (current, total) => {
+                            onProgress({
+                                phase: 'parsing',
+                                percent: Math.round(chunkBasePercent),
+                                message: `Resolving heritage (chunk ${chunkIdx + 1}/${numChunks})...`,
+                                detail: `${current}/${total} records`,
+                                stats: { filesProcessed: filesParsedSoFar, totalFiles: totalParseable, nodesCreated: graph.nodeCount },
+                            });
+                        }),
+                        processRoutesFromExtracted(graph, chunkWorkerData.routes ?? [], ctx, (current, total) => {
+                            onProgress({
+                                phase: 'parsing',
+                                percent: Math.round(chunkBasePercent),
+                                message: `Resolving routes (chunk ${chunkIdx + 1}/${numChunks})...`,
+                                detail: `${current}/${total} routes`,
+                                stats: { filesProcessed: filesParsedSoFar, totalFiles: totalParseable, nodesCreated: graph.nodeCount },
+                            });
+                        }),
+                    ]);
+                }
+                else {
+                    await processImports(graph, chunkFiles, astCache, ctx, undefined, repoPath, allPaths);
+                    sequentialChunkPaths.push(chunkPaths);
+                }
+                filesParsedSoFar += chunkFiles.length;
+                // Clear AST cache between chunks to free memory; chunk locals go out of scope for GC
+                astCache.clear();
+                // Attempt GC between chunks if under memory pressure
+                if (memoryGuard.isUnderPressure()) {
+                    memoryGuard.tryGC();
+                    if (isDev) {
+                        console.log(`⚠️ Memory pressure after chunk ${chunkIdx + 1}: ${memoryGuard.summary()}`);
+                    }
+                }
+            }
+        }
+        finally {
+            await workerPool?.terminate();
+        }
+        // Sequential fallback: re-read source for call/heritage resolution
+        for (const chunkPaths of sequentialChunkPaths) {
+            const chunkContents = await readFileContents(repoPath, chunkPaths);
+            const chunkFiles = chunkPaths
+                .filter(p => chunkContents.has(p))
+                .map(p => ({ path: p, content: chunkContents.get(p) }));
+            astCache = createASTCache(chunkFiles.length);
+            const rubyHeritage = await processCalls(graph, chunkFiles, astCache, ctx);
+            await processHeritage(graph, chunkFiles, astCache, ctx);
+            if (rubyHeritage.length > 0) {
+                await processHeritageFromExtracted(graph, rubyHeritage, ctx);
+            }
+            astCache.clear();
+        }
+        // Phase B: Resolve ALL deferred calls now that every symbol is registered
+        if (allExtractedCalls.length > 0) {
+            onProgress({
+                phase: 'parsing',
+                percent: 82,
+                message: `Resolving ${allExtractedCalls.length} calls across all files...`,
+                stats: { filesProcessed: totalFiles, totalFiles, nodesCreated: graph.nodeCount },
+            });
+            await processCallsFromExtracted(graph, allExtractedCalls, ctx, (current, total) => {
+                onProgress({
+                    phase: 'parsing',
+                    percent: 82,
+                    message: `Resolving calls: ${current}/${total} files...`,
+                    stats: { filesProcessed: totalFiles, totalFiles, nodesCreated: graph.nodeCount },
+                });
+            }, allConstructorBindings.length > 0 ? allConstructorBindings : undefined);
+        }
+        // Log resolution cache stats in dev mode
+        if (isDev) {
+            const rcStats = ctx.getStats();
+            const total = rcStats.cacheHits + rcStats.cacheMisses;
+            const hitRate = total > 0 ? ((rcStats.cacheHits / total) * 100).toFixed(1) : '0';
+            console.log(`🔍 Resolution cache: ${rcStats.cacheHits} hits, ${rcStats.cacheMisses} misses (${hitRate}% hit rate)`);
+        }
+        // Free import resolution context (~94MB+ for large repos)
+        allPathObjects.length = 0;
+        importCtx.resolveCache.clear();
+        importCtx.suffixIndex = null;
+        importCtx.normalizedFileList = null;
+        // Phase 4.5a: DI dependency edges (DEPENDS_ON from constructor params, PROVIDES from factories)
+        const [diEdgeCount, providesEdgeCount] = await Promise.all([
+            createDependsOnEdges(graph, ctx),
+            createProvidesEdges(graph, ctx),
+        ]);
+        if (isDev && (diEdgeCount > 0 || providesEdgeCount > 0)) {
+            console.log(`💉 DI: ${diEdgeCount} DEPENDS_ON edges, ${providesEdgeCount} PROVIDES edges`);
+        }
+        // Phase 4.5b: Method Resolution Order
+        onProgress({
+            phase: 'parsing',
+            percent: 81,
+            message: 'Computing method resolution order...',
+            stats: { filesProcessed: totalFiles, totalFiles, nodesCreated: graph.nodeCount },
+        });
+        const mroResult = computeMRO(graph);
+        if (isDev && mroResult.entries.length > 0) {
+            console.log(`🔀 MRO: ${mroResult.entries.length} classes analyzed, ${mroResult.ambiguityCount} ambiguities found, ${mroResult.overrideEdges} OVERRIDES edges`);
+        }
+        // Phase 5: Communities
+        onProgress({
+            phase: 'communities',
+            percent: 82,
+            message: 'Detecting code communities...',
+            stats: { filesProcessed: totalFiles, totalFiles, nodesCreated: graph.nodeCount },
+        });
+        const communityResult = await processCommunities(graph, (message, progress) => {
+            const communityProgress = 82 + (progress * 0.10);
+            onProgress({
+                phase: 'communities',
+                percent: Math.round(communityProgress),
+                message,
+                stats: { filesProcessed: totalFiles, totalFiles, nodesCreated: graph.nodeCount },
+            });
+        });
+        if (isDev) {
+            console.log(`🏘️ Community detection: ${communityResult.stats.totalCommunities} communities found (modularity: ${communityResult.stats.modularity.toFixed(3)})`);
+        }
+        communityResult.communities.forEach(comm => {
+            graph.addNode({
+                id: comm.id,
+                label: 'Community',
+                properties: {
+                    name: comm.label,
+                    filePath: '',
+                    heuristicLabel: comm.heuristicLabel,
+                    cohesion: comm.cohesion,
+                    symbolCount: comm.symbolCount,
+                }
+            });
+        });
+        communityResult.memberships.forEach(membership => {
+            graph.addRelationship({
+                id: `${membership.nodeId}_member_of_${membership.communityId}`,
+                type: 'MEMBER_OF',
+                sourceId: membership.nodeId,
+                targetId: membership.communityId,
+                confidence: 1.0,
+                reason: 'leiden-algorithm',
+            });
+        });
+        // Phase 6: Processes
+        onProgress({
+            phase: 'processes',
+            percent: 94,
+            message: 'Detecting execution flows...',
+            stats: { filesProcessed: totalFiles, totalFiles, nodesCreated: graph.nodeCount },
+        });
+        let symbolCount = 0;
+        graph.forEachNode(n => { if (n.label !== 'File')
+            symbolCount++; });
+        const dynamicMaxProcesses = Math.max(20, Math.min(300, Math.round(symbolCount / 10)));
+        const processResult = await processProcesses(graph, communityResult.memberships, (message, progress) => {
+            const processProgress = 94 + (progress * 0.05);
+            onProgress({
+                phase: 'processes',
+                percent: Math.round(processProgress),
+                message,
+                stats: { filesProcessed: totalFiles, totalFiles, nodesCreated: graph.nodeCount },
+            });
+        }, { maxProcesses: dynamicMaxProcesses, minSteps: 3 });
+        if (isDev) {
+            console.log(`🔄 Process detection: ${processResult.stats.totalProcesses} processes found (${processResult.stats.crossCommunityCount} cross-community)`);
+        }
+        processResult.processes.forEach(proc => {
+            graph.addNode({
+                id: proc.id,
+                label: 'Process',
+                properties: {
+                    name: proc.label,
+                    filePath: '',
+                    heuristicLabel: proc.heuristicLabel,
+                    processType: proc.processType,
+                    stepCount: proc.stepCount,
+                    communities: proc.communities,
+                    entryPointId: proc.entryPointId,
+                    terminalId: proc.terminalId,
+                }
+            });
+        });
+        processResult.steps.forEach(step => {
+            graph.addRelationship({
+                id: `${step.nodeId}_step_${step.step}_${step.processId}`,
+                type: 'STEP_IN_PROCESS',
+                sourceId: step.nodeId,
+                targetId: step.processId,
+                confidence: 1.0,
+                reason: 'trace-detection',
+                step: step.step,
+            });
+        });
+        onProgress({
+            phase: 'complete',
+            percent: 100,
+            message: `Graph complete! ${communityResult.stats.totalCommunities} communities, ${processResult.stats.totalProcesses} processes detected.`,
+            stats: {
+                filesProcessed: totalFiles,
+                totalFiles,
+                nodesCreated: graph.nodeCount
+            },
+        });
+        astCache.clear();
+        return { graph, repoPath, totalFileCount: totalFiles, communityResult, processResult };
+    }
+    catch (error) {
+        cleanup();
+        throw error;
+    }
+};

package/dist/core/ingestion/process-processor.d.ts

@@ -0,0 +1,42 @@
+/** @file process-processor.ts @description Detects execution flows (Processes) by finding entry points, tracing forward via confidence-gated beam search, deduplicating similar paths, and labeling with heuristic names */
+import { KnowledgeGraph } from '../graph/types.js';
+import { CommunityMembership } from './community-processor.js';
+export interface ProcessDetectionConfig {
+    maxTraceDepth: number;
+    maxProcesses: number;
+    minSteps: number;
+    /** Minimum per-edge confidence to follow during beam search */
+    minEdgeConfidence: number;
+    /** Minimum cumulative path confidence — paths below this are pruned */
+    minPathConfidence: number;
+    /** Maximum active paths in the beam (global memory bound) */
+    maxBeamSize: number;
+}
+export interface ProcessNode {
+    id: string;
+    label: string;
+    heuristicLabel: string;
+    processType: 'intra_community' | 'cross_community';
+    stepCount: number;
+    communities: string[];
+    entryPointId: string;
+    terminalId: string;
+    trace: string[];
+}
+export interface ProcessStep {
+    nodeId: string;
+    processId: string;
+    step: number;
+}
+export interface ProcessDetectionResult {
+    processes: ProcessNode[];
+    steps: ProcessStep[];
+    stats: {
+        totalProcesses: number;
+        crossCommunityCount: number;
+        avgStepCount: number;
+        entryPointsFound: number;
+    };
+}
+/** Detect processes (execution flows) in the knowledge graph, runs after community detection */
+export declare const processProcesses: (knowledgeGraph: KnowledgeGraph, memberships: CommunityMembership[], onProgress?: (message: string, progress: number) => void, config?: Partial<ProcessDetectionConfig>) => Promise<ProcessDetectionResult>;