gitnexus 1.0.0

package/dist/core/ingestion/parsing-processor.js

@@ -0,0 +1,249 @@
+import Parser from 'tree-sitter';
+import { loadParser, loadLanguage } from '../tree-sitter/parser-loader.js';
+import { LANGUAGE_QUERIES } from './tree-sitter-queries.js';
+import { generateId } from '../../lib/utils.js';
+import { getLanguageFromFilename, yieldToEventLoop } from './utils.js';
+// ============================================================================
+// EXPORT DETECTION - Language-specific visibility detection
+// ============================================================================
+/**
+ * Check if a symbol (function, class, etc.) is exported/public
+ * Handles all 9 supported languages with explicit logic
+ *
+ * @param node - The AST node for the symbol name
+ * @param name - The symbol name
+ * @param language - The programming language
+ * @returns true if the symbol is exported/public
+ */
+const isNodeExported = (node, name, language) => {
+    let current = node;
+    switch (language) {
+        // JavaScript/TypeScript: Check for export keyword in ancestors
+        case 'javascript':
+        case 'typescript':
+            while (current) {
+                const type = current.type;
+                if (type === 'export_statement' ||
+                    type === 'export_specifier' ||
+                    type === 'lexical_declaration' && current.parent?.type === 'export_statement') {
+                    return true;
+                }
+                // Also check if text starts with 'export '
+                if (current.text?.startsWith('export ')) {
+                    return true;
+                }
+                current = current.parent;
+            }
+            return false;
+        // Python: Public if no leading underscore (convention)
+        case 'python':
+            return !name.startsWith('_');
+        // Java: Check for 'public' modifier
+        // In tree-sitter Java, modifiers are siblings of the name node, not parents
+        case 'java':
+            while (current) {
+                // Check if this node or any sibling is a 'modifiers' node containing 'public'
+                if (current.parent) {
+                    const parent = current.parent;
+                    // Check all children of the parent for modifiers
+                    for (let i = 0; i < parent.childCount; i++) {
+                        const child = parent.child(i);
+                        if (child?.type === 'modifiers' && child.text?.includes('public')) {
+                            return true;
+                        }
+                    }
+                    // Also check if the parent's text starts with 'public' (fallback)
+                    if (parent.type === 'method_declaration' || parent.type === 'constructor_declaration') {
+                        if (parent.text?.trimStart().startsWith('public')) {
+                            return true;
+                        }
+                    }
+                }
+                current = current.parent;
+            }
+            return false;
+        // C#: Check for 'public' modifier in ancestors
+        case 'csharp':
+            while (current) {
+                if (current.type === 'modifier' || current.type === 'modifiers') {
+                    if (current.text?.includes('public'))
+                        return true;
+                }
+                current = current.parent;
+            }
+            return false;
+        // Go: Uppercase first letter = exported
+        case 'go':
+            if (name.length === 0)
+                return false;
+            const first = name[0];
+            // Must be uppercase letter (not a number or symbol)
+            return first === first.toUpperCase() && first !== first.toLowerCase();
+        // Rust: Check for 'pub' visibility modifier
+        case 'rust':
+            while (current) {
+                if (current.type === 'visibility_modifier') {
+                    if (current.text?.includes('pub'))
+                        return true;
+                }
+                current = current.parent;
+            }
+            return false;
+        // C/C++: No native export concept at language level
+        // Entry points will be detected via name patterns (main, etc.)
+        case 'c':
+        case 'cpp':
+            return false;
+        default:
+            return false;
+    }
+};
+export const processParsing = async (graph, files, symbolTable, astCache, onFileProgress) => {
+    const parser = await loadParser();
+    const total = files.length;
+    for (let i = 0; i < files.length; i++) {
+        const file = files[i];
+        // Report progress for each file
+        onFileProgress?.(i + 1, total, file.path);
+        // Yield to event loop periodically so spinner can update
+        if (i % 20 === 0)
+            await yieldToEventLoop();
+        const language = getLanguageFromFilename(file.path);
+        if (!language)
+            continue;
+        await loadLanguage(language, file.path);
+        // 3. Parse the text content into an AST
+        // Use larger bufferSize for files > 32KB (default limit)
+        let tree;
+        try {
+            tree = parser.parse(file.content, undefined, { bufferSize: 1024 * 256 });
+        }
+        catch (parseError) {
+            // Skip files that can't be parsed (binary, encoding issues, etc.)
+            console.warn(`Skipping unparseable file: ${file.path}`);
+            continue;
+        }
+        // Store in cache immediately (this might evict an old one)
+        astCache.set(file.path, tree);
+        // 4. Get the specific query string for this language
+        const queryString = LANGUAGE_QUERIES[language];
+        if (!queryString) {
+            continue;
+        }
+        // 5. Run the query against the AST root node
+        // This looks for patterns like (function_declaration)
+        let query;
+        let matches;
+        try {
+            const language = parser.getLanguage();
+            query = new Parser.Query(language, queryString);
+            matches = query.matches(tree.rootNode);
+        }
+        catch (queryError) {
+            console.warn(`Query error for ${file.path}:`, queryError);
+            continue;
+        }
+        // 6. Process every match found
+        matches.forEach(match => {
+            const captureMap = {};
+            match.captures.forEach(c => {
+                captureMap[c.name] = c.node;
+            });
+            // Skip imports here - they are handled by import-processor.ts
+            // which creates proper File -> IMPORTS -> File relationships
+            if (captureMap['import']) {
+                return;
+            }
+            // Skip call expressions - they are handled by call-processor.ts
+            if (captureMap['call']) {
+                return;
+            }
+            const nameNode = captureMap['name'];
+            if (!nameNode)
+                return;
+            const nodeName = nameNode.text;
+            let nodeLabel = 'CodeElement';
+            // Core types
+            if (captureMap['definition.function'])
+                nodeLabel = 'Function';
+            else if (captureMap['definition.class'])
+                nodeLabel = 'Class';
+            else if (captureMap['definition.interface'])
+                nodeLabel = 'Interface';
+            else if (captureMap['definition.method'])
+                nodeLabel = 'Method';
+            // Struct types (C, C++, Go, Rust, C#)
+            else if (captureMap['definition.struct'])
+                nodeLabel = 'Struct';
+            // Enum types
+            else if (captureMap['definition.enum'])
+                nodeLabel = 'Enum';
+            // Namespace/Module (C++, C#, Rust)
+            else if (captureMap['definition.namespace'])
+                nodeLabel = 'Namespace';
+            else if (captureMap['definition.module'])
+                nodeLabel = 'Module';
+            // Rust-specific
+            else if (captureMap['definition.trait'])
+                nodeLabel = 'Trait';
+            else if (captureMap['definition.impl'])
+                nodeLabel = 'Impl';
+            else if (captureMap['definition.type'])
+                nodeLabel = 'TypeAlias';
+            else if (captureMap['definition.const'])
+                nodeLabel = 'Const';
+            else if (captureMap['definition.static'])
+                nodeLabel = 'Static';
+            // C-specific
+            else if (captureMap['definition.typedef'])
+                nodeLabel = 'Typedef';
+            else if (captureMap['definition.macro'])
+                nodeLabel = 'Macro';
+            else if (captureMap['definition.union'])
+                nodeLabel = 'Union';
+            // C#-specific
+            else if (captureMap['definition.property'])
+                nodeLabel = 'Property';
+            else if (captureMap['definition.record'])
+                nodeLabel = 'Record';
+            else if (captureMap['definition.delegate'])
+                nodeLabel = 'Delegate';
+            // Java-specific
+            else if (captureMap['definition.annotation'])
+                nodeLabel = 'Annotation';
+            else if (captureMap['definition.constructor'])
+                nodeLabel = 'Constructor';
+            // C++ template
+            else if (captureMap['definition.template'])
+                nodeLabel = 'Template';
+            const nodeId = generateId(nodeLabel, `${file.path}:${nodeName}`);
+            const node = {
+                id: nodeId,
+                label: nodeLabel,
+                properties: {
+                    name: nodeName,
+                    filePath: file.path,
+                    startLine: nameNode.startPosition.row,
+                    endLine: nameNode.endPosition.row,
+                    language: language,
+                    isExported: isNodeExported(nameNode, nodeName, language),
+                }
+            };
+            graph.addNode(node);
+            // Register in Symbol Table (only definitions, not imports)
+            symbolTable.add(file.path, nodeName, nodeId, nodeLabel);
+            const fileId = generateId('File', file.path);
+            const relId = generateId('DEFINES', `${fileId}->${nodeId}`);
+            const relationship = {
+                id: relId,
+                sourceId: fileId,
+                targetId: nodeId,
+                type: 'DEFINES',
+                confidence: 1.0,
+                reason: '',
+            };
+            graph.addRelationship(relationship);
+        });
+        // Don't delete tree here - LRU cache handles cleanup when evicted
+    }
+};

package/dist/core/ingestion/pipeline.d.ts

@@ -0,0 +1,2 @@
+import { PipelineProgress, PipelineResult } from '../../types/pipeline.js';
+export declare const runPipelineFromRepo: (repoPath: string, onProgress: (progress: PipelineProgress) => void) => Promise<PipelineResult>;

package/dist/core/ingestion/pipeline.js

@@ -0,0 +1,228 @@
+import { createKnowledgeGraph } from '../graph/graph.js';
+import { processStructure } from './structure-processor.js';
+import { processParsing } from './parsing-processor.js';
+import { processImports, createImportMap } from './import-processor.js';
+import { processCalls } from './call-processor.js';
+import { processHeritage } from './heritage-processor.js';
+import { processCommunities } from './community-processor.js';
+import { processProcesses } from './process-processor.js';
+import { createSymbolTable } from './symbol-table.js';
+import { createASTCache } from './ast-cache.js';
+import { walkRepository } from './filesystem-walker.js';
+const isDev = process.env.NODE_ENV !== 'production';
+export const runPipelineFromRepo = async (repoPath, onProgress) => {
+    const graph = createKnowledgeGraph();
+    const fileContents = new Map();
+    const symbolTable = createSymbolTable();
+    const astCache = createASTCache(50);
+    const importMap = createImportMap();
+    const cleanup = () => {
+        astCache.clear();
+        symbolTable.clear();
+    };
+    try {
+        onProgress({
+            phase: 'extracting',
+            percent: 0,
+            message: 'Scanning repository...',
+        });
+        const files = await walkRepository(repoPath, (current, total, filePath) => {
+            const scanProgress = Math.round((current / total) * 15);
+            onProgress({
+                phase: 'extracting',
+                percent: scanProgress,
+                message: 'Scanning repository...',
+                detail: filePath,
+                stats: { filesProcessed: current, totalFiles: total, nodesCreated: graph.nodeCount },
+            });
+        });
+        files.forEach(f => fileContents.set(f.path, f.content));
+        onProgress({
+            phase: 'extracting',
+            percent: 15,
+            message: 'Repository scanned successfully',
+            stats: { filesProcessed: files.length, totalFiles: files.length, nodesCreated: graph.nodeCount },
+        });
+        onProgress({
+            phase: 'structure',
+            percent: 15,
+            message: 'Analyzing project structure...',
+            stats: { filesProcessed: 0, totalFiles: files.length, nodesCreated: graph.nodeCount },
+        });
+        const filePaths = files.map(f => f.path);
+        processStructure(graph, filePaths);
+        onProgress({
+            phase: 'structure',
+            percent: 30,
+            message: 'Project structure analyzed',
+            stats: { filesProcessed: files.length, totalFiles: files.length, nodesCreated: graph.nodeCount },
+        });
+        onProgress({
+            phase: 'parsing',
+            percent: 30,
+            message: 'Parsing code definitions...',
+            stats: { filesProcessed: 0, totalFiles: files.length, nodesCreated: graph.nodeCount },
+        });
+        await processParsing(graph, files, symbolTable, astCache, (current, total, filePath) => {
+            const parsingProgress = 30 + ((current / total) * 40);
+            onProgress({
+                phase: 'parsing',
+                percent: Math.round(parsingProgress),
+                message: 'Parsing code definitions...',
+                detail: filePath,
+                stats: { filesProcessed: current, totalFiles: total, nodesCreated: graph.nodeCount },
+            });
+        });
+        onProgress({
+            phase: 'imports',
+            percent: 70,
+            message: 'Resolving imports...',
+            stats: { filesProcessed: 0, totalFiles: files.length, nodesCreated: graph.nodeCount },
+        });
+        await processImports(graph, files, astCache, importMap, (current, total) => {
+            const importProgress = 70 + ((current / total) * 12);
+            onProgress({
+                phase: 'imports',
+                percent: Math.round(importProgress),
+                message: 'Resolving imports...',
+                stats: { filesProcessed: current, totalFiles: total, nodesCreated: graph.nodeCount },
+            });
+        }, repoPath);
+        if (isDev) {
+            const importsCount = graph.relationships.filter(r => r.type === 'IMPORTS').length;
+            console.log(`📊 Pipeline: After import phase, graph has ${importsCount} IMPORTS relationships (total: ${graph.relationshipCount})`);
+        }
+        onProgress({
+            phase: 'calls',
+            percent: 82,
+            message: 'Tracing function calls...',
+            stats: { filesProcessed: 0, totalFiles: files.length, nodesCreated: graph.nodeCount },
+        });
+        await processCalls(graph, files, astCache, symbolTable, importMap, (current, total) => {
+            const callProgress = 82 + ((current / total) * 10);
+            onProgress({
+                phase: 'calls',
+                percent: Math.round(callProgress),
+                message: 'Tracing function calls...',
+                stats: { filesProcessed: current, totalFiles: total, nodesCreated: graph.nodeCount },
+            });
+        });
+        onProgress({
+            phase: 'heritage',
+            percent: 92,
+            message: 'Extracting class inheritance...',
+            stats: { filesProcessed: 0, totalFiles: files.length, nodesCreated: graph.nodeCount },
+        });
+        await processHeritage(graph, files, astCache, symbolTable, (current, total) => {
+            const heritageProgress = 88 + ((current / total) * 4);
+            onProgress({
+                phase: 'heritage',
+                percent: Math.round(heritageProgress),
+                message: 'Extracting class inheritance...',
+                stats: { filesProcessed: current, totalFiles: total, nodesCreated: graph.nodeCount },
+            });
+        });
+        onProgress({
+            phase: 'communities',
+            percent: 92,
+            message: 'Detecting code communities...',
+            stats: { filesProcessed: files.length, totalFiles: files.length, nodesCreated: graph.nodeCount },
+        });
+        const communityResult = await processCommunities(graph, (message, progress) => {
+            const communityProgress = 92 + (progress * 0.06);
+            onProgress({
+                phase: 'communities',
+                percent: Math.round(communityProgress),
+                message,
+                stats: { filesProcessed: files.length, totalFiles: files.length, nodesCreated: graph.nodeCount },
+            });
+        });
+        if (isDev) {
+            console.log(`🏘️ Community detection: ${communityResult.stats.totalCommunities} communities found (modularity: ${communityResult.stats.modularity.toFixed(3)})`);
+        }
+        communityResult.communities.forEach(comm => {
+            graph.addNode({
+                id: comm.id,
+                label: 'Community',
+                properties: {
+                    name: comm.label,
+                    filePath: '',
+                    heuristicLabel: comm.heuristicLabel,
+                    cohesion: comm.cohesion,
+                    symbolCount: comm.symbolCount,
+                }
+            });
+        });
+        communityResult.memberships.forEach(membership => {
+            graph.addRelationship({
+                id: `${membership.nodeId}_member_of_${membership.communityId}`,
+                type: 'MEMBER_OF',
+                sourceId: membership.nodeId,
+                targetId: membership.communityId,
+                confidence: 1.0,
+                reason: 'leiden-algorithm',
+            });
+        });
+        onProgress({
+            phase: 'processes',
+            percent: 98,
+            message: 'Detecting execution flows...',
+            stats: { filesProcessed: files.length, totalFiles: files.length, nodesCreated: graph.nodeCount },
+        });
+        const processResult = await processProcesses(graph, communityResult.memberships, (message, progress) => {
+            const processProgress = 98 + (progress * 0.01);
+            onProgress({
+                phase: 'processes',
+                percent: Math.round(processProgress),
+                message,
+                stats: { filesProcessed: files.length, totalFiles: files.length, nodesCreated: graph.nodeCount },
+            });
+        });
+        if (isDev) {
+            console.log(`🔄 Process detection: ${processResult.stats.totalProcesses} processes found (${processResult.stats.crossCommunityCount} cross-community)`);
+        }
+        processResult.processes.forEach(proc => {
+            graph.addNode({
+                id: proc.id,
+                label: 'Process',
+                properties: {
+                    name: proc.label,
+                    filePath: '',
+                    heuristicLabel: proc.heuristicLabel,
+                    processType: proc.processType,
+                    stepCount: proc.stepCount,
+                    communities: proc.communities,
+                    entryPointId: proc.entryPointId,
+                    terminalId: proc.terminalId,
+                }
+            });
+        });
+        processResult.steps.forEach(step => {
+            graph.addRelationship({
+                id: `${step.nodeId}_step_${step.step}_${step.processId}`,
+                type: 'STEP_IN_PROCESS',
+                sourceId: step.nodeId,
+                targetId: step.processId,
+                confidence: 1.0,
+                reason: 'trace-detection',
+                step: step.step,
+            });
+        });
+        onProgress({
+            phase: 'complete',
+            percent: 100,
+            message: `Graph complete! ${communityResult.stats.totalCommunities} communities, ${processResult.stats.totalProcesses} processes detected.`,
+            stats: {
+                filesProcessed: files.length,
+                totalFiles: files.length,
+                nodesCreated: graph.nodeCount
+            },
+        });
+        astCache.clear();
+        return { graph, fileContents, communityResult, processResult };
+    }
+    catch (error) {
+        cleanup();
+        throw error;
+    }
+};

package/dist/core/ingestion/process-processor.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Process Detection Processor
+ *
+ * Detects execution flows (Processes) in the code graph by:
+ * 1. Finding entry points (functions with no internal callers)
+ * 2. Tracing forward via CALLS edges (BFS)
+ * 3. Grouping and deduplicating similar paths
+ * 4. Labeling with heuristic names
+ *
+ * Processes help agents understand how features work through the codebase.
+ */
+import { KnowledgeGraph } from '../graph/types.js';
+import { CommunityMembership } from './community-processor.js';
+export interface ProcessDetectionConfig {
+    maxTraceDepth: number;
+    maxBranching: number;
+    maxProcesses: number;
+    minSteps: number;
+}
+export interface ProcessNode {
+    id: string;
+    label: string;
+    heuristicLabel: string;
+    processType: 'intra_community' | 'cross_community';
+    stepCount: number;
+    communities: string[];
+    entryPointId: string;
+    terminalId: string;
+    trace: string[];
+}
+export interface ProcessStep {
+    nodeId: string;
+    processId: string;
+    step: number;
+}
+export interface ProcessDetectionResult {
+    processes: ProcessNode[];
+    steps: ProcessStep[];
+    stats: {
+        totalProcesses: number;
+        crossCommunityCount: number;
+        avgStepCount: number;
+        entryPointsFound: number;
+    };
+}
+/**
+ * Detect processes (execution flows) in the knowledge graph
+ *
+ * This runs AFTER community detection, using CALLS edges to trace flows.
+ */
+export declare const processProcesses: (knowledgeGraph: KnowledgeGraph, memberships: CommunityMembership[], onProgress?: (message: string, progress: number) => void, config?: Partial<ProcessDetectionConfig>) => Promise<ProcessDetectionResult>;