code-graph-context 1.1.0 → 2.0.0

package/dist/core/parsers/workspace-parser.js

@@ -0,0 +1,553 @@
+/**
+ * Workspace Parser
+ * Orchestrates parsing of multi-package monorepos
+ */
+import crypto from 'crypto';
+import path from 'path';
+import { glob } from 'glob';
+import { debugLog } from '../utils/file-utils.js';
+import { CORE_TYPESCRIPT_SCHEMA, } from '../config/schema.js';
+import { createFrameworkEdgeData } from '../utils/edge-factory.js';
+import { resolveProjectId } from '../utils/project-id.js';
+import { ParserFactory } from './parser-factory.js';
+export class WorkspaceParser {
+    config;
+    projectId;
+    projectType;
+    lazyLoad;
+    discoveredFiles = null;
+    parsedNodes = new Map();
+    parsedEdges = new Map();
+    accumulatedDeferredEdges = [];
+    // Shared context across all packages for cross-package edge detection
+    sharedContext = new Map();
+    // Lightweight node copies for cross-package edge detection (no AST references)
+    accumulatedParsedNodes = new Map();
+    // Framework schemas detected from packages (for edge enhancements)
+    frameworkSchemas = [];
+    constructor(config, projectId, lazyLoad = true, projectType = 'auto') {
+        this.config = config;
+        this.projectId = resolveProjectId(config.rootPath, projectId);
+        this.lazyLoad = lazyLoad;
+        this.projectType = projectType;
+    }
+    /**
+     * Get the project ID for this workspace
+     */
+    getProjectId() {
+        return this.projectId;
+    }
+    /**
+     * Get workspace configuration
+     */
+    getConfig() {
+        return this.config;
+    }
+    /**
+     * Discover all source files across all packages
+     */
+    async discoverSourceFiles() {
+        if (this.discoveredFiles !== null) {
+            // Return flattened list
+            return Array.from(this.discoveredFiles.values()).flat();
+        }
+        this.discoveredFiles = new Map();
+        let totalFiles = 0;
+        const packageCounts = {};
+        for (const pkg of this.config.packages) {
+            const files = await this.discoverPackageFiles(pkg);
+            this.discoveredFiles.set(pkg.name, files);
+            totalFiles += files.length;
+            packageCounts[pkg.name] = files.length;
+        }
+        await debugLog('WorkspaceParser discovered files', {
+            totalFiles,
+            packageCount: this.config.packages.length,
+            packageCounts,
+        });
+        return Array.from(this.discoveredFiles.values()).flat();
+    }
+    /**
+     * Discover files in a single package
+     */
+    async discoverPackageFiles(pkg) {
+        // Include both .ts and .tsx files
+        const pattern = path.join(pkg.path, '**/*.{ts,tsx}');
+        const files = await glob(pattern, {
+            ignore: ['**/node_modules/**', '**/*.d.ts', '**/dist/**', '**/build/**'],
+            absolute: true,
+        });
+        return files;
+    }
+    /**
+     * Get files grouped by package
+     */
+    async getFilesByPackage() {
+        if (this.discoveredFiles === null) {
+            await this.discoverSourceFiles();
+        }
+        return this.discoveredFiles;
+    }
+    /**
+     * Create a parser for a package using ParserFactory (supports auto-detection)
+     * Injects the shared context so context is shared across all packages.
+     */
+    async createParserForPackage(pkg) {
+        const tsConfigPath = pkg.tsConfigPath || path.join(pkg.path, 'tsconfig.json');
+        let parser;
+        if (this.projectType === 'auto') {
+            // Auto-detect framework for this specific package
+            parser = await ParserFactory.createParserWithAutoDetection(pkg.path, tsConfigPath, this.projectId, this.lazyLoad);
+        }
+        else {
+            // Use the specified project type for all packages
+            parser = ParserFactory.createParser({
+                workspacePath: pkg.path,
+                tsConfigPath,
+                projectType: this.projectType,
+                projectId: this.projectId,
+                lazyLoad: this.lazyLoad,
+            });
+        }
+        // Inject shared context so all packages share the same context
+        // This enables cross-package edge detection (e.g., INTERNAL_API_CALL)
+        parser.setSharedContext(this.sharedContext);
+        // Defer edge enhancements to WorkspaceParser's final pass
+        // This avoids duplicate work and enables cross-package edge detection
+        parser.setDeferEdgeEnhancements(true);
+        return parser;
+    }
+    /**
+     * Parse a single package and return its results
+     */
+    async parsePackage(pkg) {
+        console.log(`\nParsing package: ${pkg.name}`);
+        const parser = await this.createParserForPackage(pkg);
+        // Discover files for this package
+        const files = await this.discoverPackageFiles(pkg);
+        if (files.length === 0) {
+            console.log(`   ⚠️ No TypeScript files found in ${pkg.name}`);
+            return { nodes: [], edges: [] };
+        }
+        console.log(`   📄 ${files.length} files to parse`);
+        // Parse all files in this package
+        const result = await parser.parseChunk(files, true); // Skip edge resolution for now
+        // Add package name to all nodes
+        for (const node of result.nodes) {
+            node.properties.packageName = pkg.name;
+        }
+        console.log(`   ✅ ${result.nodes.length} nodes, ${result.edges.length} edges`);
+        return result;
+    }
+    /**
+     * Parse a chunk of files (for streaming compatibility)
+     * Files are grouped by package and parsed together
+     */
+    async parseChunk(filePaths, skipEdgeResolution = false) {
+        // Group files by package
+        const filesByPackage = new Map();
+        for (const filePath of filePaths) {
+            const pkg = this.findPackageForFile(filePath);
+            if (pkg) {
+                const files = filesByPackage.get(pkg) ?? [];
+                files.push(filePath);
+                filesByPackage.set(pkg, files);
+            }
+        }
+        const allNodes = [];
+        const allEdges = [];
+        // Parse each package's files
+        for (const [pkg, files] of filesByPackage) {
+            try {
+                const parser = await this.createParserForPackage(pkg);
+                const result = await parser.parseChunk(files, skipEdgeResolution);
+                // Add package name to nodes
+                for (const node of result.nodes) {
+                    node.properties.packageName = pkg.name;
+                }
+                // Export and accumulate deferred edges for cross-package resolution
+                const chunkData = parser.exportChunkResults();
+                this.accumulatedDeferredEdges.push(...chunkData.deferredEdges);
+                // Accumulate LIGHTWEIGHT copies of ParsedNodes for cross-package edge detection
+                // Only stores what's needed for detection patterns - NO AST references
+                const innerParsedNodes = parser.getParsedNodes();
+                for (const [nodeId, parsedNode] of innerParsedNodes) {
+                    this.accumulatedParsedNodes.set(nodeId, {
+                        id: parsedNode.id,
+                        semanticType: parsedNode.semanticType,
+                        properties: {
+                            name: parsedNode.properties.name,
+                            context: parsedNode.properties.context, // Contains propertyTypes
+                        },
+                    });
+                }
+                // Accumulate framework schemas (deduplicated by name)
+                for (const schema of parser.getFrameworkSchemas()) {
+                    if (!this.frameworkSchemas.some((s) => s.name === schema.name)) {
+                        this.frameworkSchemas.push(schema);
+                    }
+                }
+                allNodes.push(...result.nodes);
+                allEdges.push(...result.edges);
+            }
+            catch (error) {
+                console.warn(`⚠️ Failed to parse package ${pkg.name}:`, error);
+                // Continue with other packages
+            }
+        }
+        return { nodes: allNodes, edges: allEdges };
+    }
+    /**
+     * Find which package a file belongs to
+     */
+    findPackageForFile(filePath) {
+        for (const pkg of this.config.packages) {
+            if (filePath.startsWith(pkg.path)) {
+                return pkg;
+            }
+        }
+        return null;
+    }
+    /**
+     * Parse all packages in the workspace
+     */
+    async parseAll() {
+        const packageResults = new Map();
+        const allNodes = [];
+        const allEdges = [];
+        for (const pkg of this.config.packages) {
+            const result = await this.parsePackage(pkg);
+            allNodes.push(...result.nodes);
+            allEdges.push(...result.edges);
+            packageResults.set(pkg.name, {
+                nodes: result.nodes.length,
+                edges: result.edges.length,
+            });
+        }
+        console.log(`\n🎉 Workspace parsing complete!`);
+        console.log(`   Total: ${allNodes.length} nodes, ${allEdges.length} edges`);
+        return {
+            nodes: allNodes,
+            edges: allEdges,
+            packageResults,
+        };
+    }
+    /**
+     * Clear parsed data (for memory management)
+     * Note: Does NOT clear accumulated deferred edges - those need to be resolved at the end
+     */
+    clearParsedData() {
+        this.parsedNodes.clear();
+        this.parsedEdges.clear();
+    }
+    /**
+     * Add existing nodes for cross-package edge resolution
+     */
+    addExistingNodesFromChunk(nodes) {
+        for (const node of nodes) {
+            this.parsedNodes.set(node.id, node);
+        }
+    }
+    /**
+     * Get current counts for progress reporting
+     */
+    getCurrentCounts() {
+        return {
+            nodes: this.parsedNodes.size,
+            edges: this.parsedEdges.size,
+            deferredEdges: this.accumulatedDeferredEdges.length,
+        };
+    }
+    /**
+     * Resolve accumulated deferred edges against all parsed nodes
+     * Call this after all chunks have been parsed
+     */
+    async resolveDeferredEdgesManually() {
+        const resolvedEdges = [];
+        const unresolvedImports = [];
+        const unresolvedExtends = [];
+        const unresolvedImplements = [];
+        // Count by edge type for logging
+        const importsCount = this.accumulatedDeferredEdges.filter((e) => e.edgeType === 'IMPORTS').length;
+        const extendsCount = this.accumulatedDeferredEdges.filter((e) => e.edgeType === 'EXTENDS').length;
+        const implementsCount = this.accumulatedDeferredEdges.filter((e) => e.edgeType === 'IMPLEMENTS').length;
+        for (const deferred of this.accumulatedDeferredEdges) {
+            // Find target node by name, type, and optionally file path from accumulated nodes
+            const targetNode = this.findNodeByNameAndType(deferred.targetName, deferred.targetType, deferred.targetFilePath);
+            if (targetNode) {
+                // Find source node to get filePath
+                const sourceNode = this.parsedNodes.get(deferred.sourceNodeId);
+                const filePath = sourceNode?.properties.filePath ?? '';
+                // Get relationship weight from core schema
+                const coreEdgeType = deferred.edgeType;
+                const coreEdgeSchema = CORE_TYPESCRIPT_SCHEMA.edgeTypes[coreEdgeType];
+                const relationshipWeight = coreEdgeSchema?.relationshipWeight ?? 0.5;
+                // Generate a unique edge ID
+                const edgeHash = crypto
+                    .createHash('md5')
+                    .update(`${deferred.sourceNodeId}-${deferred.edgeType}-${targetNode.id}`)
+                    .digest('hex')
+                    .substring(0, 12);
+                const edge = {
+                    id: `${this.projectId}:${deferred.edgeType}:${edgeHash}`,
+                    type: deferred.edgeType,
+                    startNodeId: deferred.sourceNodeId,
+                    endNodeId: targetNode.id,
+                    properties: {
+                        coreType: coreEdgeType,
+                        projectId: this.projectId,
+                        source: 'ast',
+                        confidence: 1.0,
+                        relationshipWeight,
+                        filePath,
+                        createdAt: new Date().toISOString(),
+                    },
+                };
+                resolvedEdges.push(edge);
+            }
+            else {
+                // Track unresolved by type
+                if (deferred.edgeType === 'IMPORTS') {
+                    unresolvedImports.push(deferred.targetName);
+                }
+                else if (deferred.edgeType === 'EXTENDS') {
+                    unresolvedExtends.push(deferred.targetName);
+                }
+                else if (deferred.edgeType === 'IMPLEMENTS') {
+                    unresolvedImplements.push(deferred.targetName);
+                }
+            }
+        }
+        // Log resolution stats
+        const importsResolved = resolvedEdges.filter((e) => e.type === 'IMPORTS').length;
+        const extendsResolved = resolvedEdges.filter((e) => e.type === 'EXTENDS').length;
+        const implementsResolved = resolvedEdges.filter((e) => e.type === 'IMPLEMENTS').length;
+        debugLog('WorkspaceParser edge resolution', {
+            totalDeferredEdges: this.accumulatedDeferredEdges.length,
+            totalNodesAvailable: this.parsedNodes.size,
+            imports: {
+                queued: importsCount,
+                resolved: importsResolved,
+                unresolved: unresolvedImports.length,
+                sample: unresolvedImports.slice(0, 10),
+            },
+            extends: {
+                queued: extendsCount,
+                resolved: extendsResolved,
+                unresolved: unresolvedExtends.length,
+                sample: unresolvedExtends.slice(0, 10),
+            },
+            implements: {
+                queued: implementsCount,
+                resolved: implementsResolved,
+                unresolved: unresolvedImplements.length,
+                sample: unresolvedImplements.slice(0, 10),
+            },
+        });
+        // Clear accumulated deferred edges after resolution
+        this.accumulatedDeferredEdges = [];
+        return resolvedEdges;
+    }
+    /**
+     * Apply edge enhancements on all accumulated nodes across all packages.
+     * This enables cross-package edge detection (e.g., INTERNAL_API_CALL between services and
+     * vendor controllers in different packages).
+     *
+     * Uses shared context and accumulated ParsedNodes from all packages.
+     * @returns New edges created by edge enhancements
+     */
+    async applyEdgeEnhancementsManually() {
+        if (this.accumulatedParsedNodes.size === 0) {
+            console.log('WorkspaceParser: No accumulated nodes for edge enhancements');
+            return [];
+        }
+        if (this.frameworkSchemas.length === 0) {
+            console.log('WorkspaceParser: No framework schemas for edge enhancements');
+            return [];
+        }
+        console.log(`WorkspaceParser: Applying edge enhancements on ${this.accumulatedParsedNodes.size} accumulated nodes across all packages...`);
+        // Pre-index nodes by semantic type for O(1) lookups
+        const nodesBySemanticType = new Map();
+        for (const [nodeId, node] of this.accumulatedParsedNodes) {
+            const semanticType = node.semanticType || 'unknown';
+            if (!nodesBySemanticType.has(semanticType)) {
+                nodesBySemanticType.set(semanticType, new Map());
+            }
+            nodesBySemanticType.get(semanticType).set(nodeId, node);
+        }
+        const typeCounts = {};
+        for (const [type, nodes] of nodesBySemanticType) {
+            typeCounts[type] = nodes.size;
+        }
+        console.log(`Node distribution by semantic type:`, typeCounts);
+        const newEdges = [];
+        const edgeCountBefore = this.parsedEdges.size;
+        // Apply edge enhancements from all framework schemas
+        for (const frameworkSchema of this.frameworkSchemas) {
+            for (const edgeEnhancement of Object.values(frameworkSchema.edgeEnhancements)) {
+                const enhancementEdges = await this.applyEdgeEnhancement(edgeEnhancement, nodesBySemanticType);
+                newEdges.push(...enhancementEdges);
+            }
+        }
+        const newEdgeCount = this.parsedEdges.size - edgeCountBefore;
+        console.log(`Created ${newEdgeCount} cross-package edges from edge enhancements`);
+        return newEdges;
+    }
+    /**
+     * Apply a single edge enhancement across all accumulated parsed nodes.
+     * Uses LightweightParsedNode which contains only fields needed for detection:
+     * - id, semanticType, properties.context (with propertyTypes)
+     * Detection patterns must NOT access sourceNode (AST) - use properties instead.
+     */
+    async applyEdgeEnhancement(edgeEnhancement, _nodesBySemanticType) {
+        const newEdges = [];
+        // Track created edges with simple key to avoid duplicate hash computations
+        const createdEdgeKeys = new Set();
+        try {
+            // For now, iterate all nodes. Detection pattern short-circuits on semantic type.
+            // Future optimization: use _nodesBySemanticType to only iterate relevant pairs.
+            const allTargetNodes = new Map([...this.accumulatedParsedNodes]);
+            for (const [sourceId, sourceNode] of this.accumulatedParsedNodes) {
+                for (const [targetId, targetNode] of allTargetNodes) {
+                    if (sourceId === targetId)
+                        continue;
+                    // Run detection pattern FIRST (cheap semantic type checks)
+                    if (edgeEnhancement.detectionPattern(sourceNode, targetNode, this.accumulatedParsedNodes, this.sharedContext)) {
+                        const simpleKey = `${sourceId}:${targetId}`;
+                        if (createdEdgeKeys.has(simpleKey))
+                            continue;
+                        createdEdgeKeys.add(simpleKey);
+                        // Extract context for this edge
+                        let context = {};
+                        if (edgeEnhancement.contextExtractor) {
+                            context = edgeEnhancement.contextExtractor(sourceNode, targetNode, this.accumulatedParsedNodes, this.sharedContext);
+                        }
+                        const edge = this.createFrameworkEdge(edgeEnhancement.semanticType, edgeEnhancement.neo4j.relationshipType, sourceId, targetId, context, edgeEnhancement.relationshipWeight);
+                        this.parsedEdges.set(edge.id, edge);
+                        newEdges.push(edge);
+                    }
+                }
+            }
+        }
+        catch (error) {
+            console.error(`Error applying edge enhancement ${edgeEnhancement.name}:`, error);
+        }
+        return newEdges;
+    }
+    /**
+     * Create a framework edge with semantic type and properties.
+     */
+    createFrameworkEdge(semanticType, relationshipType, sourceId, targetId, context, relationshipWeight) {
+        const { id, properties } = createFrameworkEdgeData({
+            semanticType,
+            sourceNodeId: sourceId,
+            targetNodeId: targetId,
+            projectId: this.projectId,
+            context,
+            relationshipWeight,
+        });
+        return {
+            id,
+            type: relationshipType,
+            startNodeId: sourceId,
+            endNodeId: targetId,
+            properties,
+        };
+    }
+    /**
+     * Find a node by name and type from accumulated nodes
+     * For SourceFiles, implements smart import resolution:
+     * - Direct file path match
+     * - Relative import resolution (./foo, ../bar)
+     * - Scoped package imports (@workspace/ui, @ui/core)
+     *
+     * For ClassDeclaration/InterfaceDeclaration with filePath, uses precise matching.
+     */
+    findNodeByNameAndType(name, type, filePath) {
+        const allNodes = [...this.parsedNodes.values()];
+        // If we have a file path and it's not a SourceFile, use precise matching first
+        if (filePath && type !== 'SourceFile') {
+            for (const node of allNodes) {
+                if (node.properties.coreType === type &&
+                    node.properties.name === name &&
+                    node.properties.filePath === filePath) {
+                    return node;
+                }
+            }
+            // If precise match fails, fall through to name-only matching below
+        }
+        // For SOURCE_FILE with import specifier, try multiple matching strategies
+        if (type === 'SourceFile') {
+            // Strategy 1: Direct file path match
+            for (const node of allNodes) {
+                if (node.labels.includes(type) && node.properties.filePath === name) {
+                    return node;
+                }
+            }
+            // Strategy 2: Resolve relative imports (./foo, ../bar)
+            if (name.startsWith('.')) {
+                // Normalize: remove leading ./ or ../
+                const normalizedPath = name.replace(/^\.\.\//, '').replace(/^\.\//, '');
+                // Try matching with common extensions
+                const extensions = ['', '.ts', '.tsx', '/index.ts', '/index.tsx'];
+                for (const ext of extensions) {
+                    const searchPath = normalizedPath + ext;
+                    for (const node of allNodes) {
+                        if (node.labels.includes(type)) {
+                            // Match if filePath ends with the normalized path
+                            if (node.properties.filePath.endsWith(searchPath) ||
+                                node.properties.filePath.endsWith('/' + searchPath)) {
+                                return node;
+                            }
+                        }
+                    }
+                }
+            }
+            // Strategy 3: Workspace package imports (@workspace/ui, @ui/core)
+            if (name.startsWith('@')) {
+                const parts = name.split('/');
+                const packageName = parts.slice(0, 2).join('/'); // @scope/package
+                const subPath = parts.slice(2).join('/'); // rest of path after package name
+                // First, try to find an exact match with subpath
+                if (subPath) {
+                    const extensions = ['', '.ts', '.tsx', '/index.ts', '/index.tsx'];
+                    for (const ext of extensions) {
+                        const searchPath = subPath + ext;
+                        for (const node of allNodes) {
+                            if (node.labels.includes(type) && node.properties.packageName === packageName) {
+                                if (node.properties.filePath.endsWith(searchPath) ||
+                                    node.properties.filePath.endsWith('/' + searchPath)) {
+                                    return node;
+                                }
+                            }
+                        }
+                    }
+                }
+                // For bare package imports (@workspace/ui), look for index files
+                if (!subPath) {
+                    for (const node of allNodes) {
+                        if (node.labels.includes(type) && node.properties.packageName === packageName) {
+                            const fileName = node.properties.name;
+                            if (fileName === 'index.ts' || fileName === 'index.tsx') {
+                                return node;
+                            }
+                        }
+                    }
+                    // If no index file, return any file from the package as a fallback
+                    for (const node of allNodes) {
+                        if (node.labels.includes(type) && node.properties.packageName === packageName) {
+                            return node;
+                        }
+                    }
+                }
+            }
+        }
+        // Default: exact name match (for non-SourceFile types like classes, interfaces)
+        for (const node of allNodes) {
+            if (node.properties.coreType === type && node.properties.name === name) {
+                return node;
+            }
+        }
+        return undefined;
+    }
+}

package/dist/core/utils/edge-factory.js

@@ -0,0 +1,37 @@
+/**
+ * Edge Factory
+ * Shared utilities for creating framework edges with consistent ID generation and properties
+ */
+import crypto from 'crypto';
+/**
+ * Generate a deterministic edge ID based on semantic type, source, and target.
+ * Uses SHA256 hash truncated to 16 characters for uniqueness.
+ */
+export const generateFrameworkEdgeId = (semanticType, sourceNodeId, targetNodeId) => {
+    const edgeIdentity = `${semanticType}::${sourceNodeId}::${targetNodeId}`;
+    const edgeHash = crypto.createHash('sha256').update(edgeIdentity).digest('hex').substring(0, 16);
+    return `${semanticType}:${edgeHash}`;
+};
+/**
+ * Create framework edge ID and properties.
+ * Returns common edge data that can be used to construct either ParsedEdge or Neo4jEdge.
+ *
+ * @param params - Edge parameters
+ * @returns Edge ID and properties object
+ */
+export const createFrameworkEdgeData = (params) => {
+    const { semanticType, sourceNodeId, targetNodeId, projectId, context = {}, relationshipWeight = 0.5 } = params;
+    const id = generateFrameworkEdgeId(semanticType, sourceNodeId, targetNodeId);
+    const properties = {
+        coreType: semanticType,
+        projectId,
+        semanticType,
+        source: 'pattern',
+        confidence: 0.8,
+        relationshipWeight,
+        filePath: '',
+        createdAt: new Date().toISOString(),
+        context,
+    };
+    return { id, properties };
+};