arcvision 0.2.3 → 0.2.4

package/src/core/scanner.js

@@ -1,518 +0,0 @@
-const { glob } = require('glob');
-const path = require('path');
-const fs = require('fs');
-const parser = require('./parser-enhanced'); // Use enhanced parser
-const pluginManager = require('../plugins/plugin-manager');
-const { loadTSConfig } = require('./tsconfig-utils');
-const { resolveImport } = require('./path-resolver');
-const { buildReverseDependencyGraph, computeBlastRadius } = require('./blastRadius');
-
-// Import new engine modules
-const { buildContext } = require('../engine/context_builder');
-const { validateContext } = require('../engine/context_validator');
-const { sortContext } = require('../engine/context_sorter');
-
-// Import semantic analyzer for cross-file analysis
-const { analyzeSemantics } = require('./semantic-analyzer');
-
-async function scan(directory) {
-    // Normalize helper
-    const normalize = p => p.replace(/\\/g, '/');
-
-    const options = {
-        ignore: ['**/node_modules/**', '**/.git/**', '**/dist/**', '**/build/**'],
-        cwd: directory,
-        absolute: true
-    };
-
-    try {
-        // Load plugins
-        const pluginDir = path.join(__dirname, '../plugins');
-        pluginManager.loadPluginsFromDirectory(pluginDir);
-
-        const files = await glob('**/*.{js,jsx,ts,tsx,json}', { ...options, ignore: [...options.ignore, '**/*.d.ts', '**/node_modules/**', '**/.git/**', '**/dist/**', '**/build/**', '**/.next/**', '**/coverage/**'] });
-
-        const architectureMap = {
-            nodes: [],
-            edges: []
-        };
-
-        const fileMap = new Map();
-        let totalImportsFound = 0; // Track total imports found
-
-        // Process files with plugins
-        for (const file of files) {
-            try {
-                const relativePath = path.relative(directory, file);
-                const normalizedRelativePath = normalize(relativePath);
-
-                // Handle different file types appropriately
-                let metadata;
-
-                if (file.endsWith('.json')) {
-                    // Handle JSON files separately
-                    try {
-                        const content = fs.readFileSync(file, 'utf-8');
-                        JSON.parse(content); // Validate it's valid JSON
-                        metadata = {
-                            id: file,
-                            imports: [],
-                            exports: [],
-                            functions: [],
-                            apiCalls: [],
-                            isJson: true
-                        };
-                    } catch (jsonError) {
-                        // If JSON is invalid, skip this file
-                        console.warn(`⚠️ Invalid JSON in ${file} — file skipped`);
-                        continue;
-                    }
-                } else {
-                    // Parse code files
-                    metadata = parser.parseFile(file);
-                }
-
-                // Process with plugins
-                metadata = await pluginManager.processFile(file, metadata);
-
-                // Count imports found in this file
-                if (metadata.imports && Array.isArray(metadata.imports)) {
-                    totalImportsFound += metadata.imports.length;
-                }
-
-                const node = {
-                    id: normalizedRelativePath,
-                    type: 'file',
-                    metadata: metadata
-                };
-
-                architectureMap.nodes.push(node);
-                fileMap.set(normalizedRelativePath, metadata);
-            } catch (e) {
-                // Log a clear, user-friendly warning message and continue scanning
-                console.warn(`⚠️ Failed to process ${file} — file skipped, you should check manually (${e.message})`);
-            }
-        }
-
-        // NEW: Perform semantic analysis to build cross-file usage edges
-        console.log('Performing semantic analysis...');
-        const semanticResults = analyzeSemantics(architectureMap.nodes);
-        console.log(`Semantic analysis complete: ${semanticResults.stats.totalSymbols} symbols, ${semanticResults.stats.totalUsages} usage edges`);
-
-        // Load tsconfig for path resolution
-        const tsconfig = loadTSConfig(directory);
-
-        // Create a mapping of all possible normalized paths that could match imports
-        // This includes both the direct file paths and possible variations
-        const allPossiblePaths = new Set();
-        const pathToNodeIdMap = new Map(); // This maps normalized paths to node IDs
-
-        architectureMap.nodes.forEach(node => {
-            const normalizedPath = normalize(node.id);
-            allPossiblePaths.add(normalizedPath);
-            pathToNodeIdMap.set(normalizedPath, node.id);
-        });
-
-        // Process imports to create edges
-        let unresolvedImports = 0;
-        let resolvedImports = 0;
-        architectureMap.nodes.forEach(node => {
-            if (node.metadata.imports && Array.isArray(node.metadata.imports)) {
-                node.metadata.imports.forEach(imp => {
-                    if (imp.source && typeof imp.source === 'string') {
-                        let targetFound = false;
-
-                        try {
-                            // First, check if imp.source directly matches any node path (exact match)
-                            if (allPossiblePaths.has(imp.source)) {
-                                architectureMap.edges.push({
-                                    source: normalize(node.id),
-                                    target: imp.source,
-                                    type: imp.type || 'import'
-                                });
-                                resolvedImports++;
-                                targetFound = true;
-                            }
-
-                            if (!targetFound) {
-                                // Try to resolve the import path using the resolver
-                                const resolvedPath = resolveImport(
-                                    imp.source,
-                                    path.join(directory, node.id),
-                                    directory,
-                                    tsconfig
-                                );
-
-                                if (resolvedPath) {
-                                    // Convert resolved absolute path back to relative path
-                                    const relativeResolvedPath = path.relative(directory, resolvedPath);
-                                    const normalizedResolvedPath = normalize(relativeResolvedPath);
-
-                                    // Check if the resolved file exists in our scanned files
-                                    if (allPossiblePaths.has(normalizedResolvedPath)) {
-                                        architectureMap.edges.push({
-                                            source: normalize(node.id),
-                                            target: normalizedResolvedPath,
-                                            type: imp.type || 'import'
-                                        });
-                                        resolvedImports++;
-                                        targetFound = true;
-                                    }
-                                }
-                            }
-
-                            if (!targetFound) {
-                                // As a fallback, check if the import path could match by simple relative path calculation
-                                // For example, if node is 'src/core/scanner.js' and import is './parser.js',
-                                // it should resolve to 'src/core/parser.js'
-                                try {
-                                    const baseDir = path.dirname(path.join(directory, node.id));
-                                    const calculatedAbsolutePath = path.resolve(baseDir, imp.source);
-                                    const calculatedRelativePath = path.relative(directory, calculatedAbsolutePath);
-                                    const calculatedNormalizedPath = normalize(calculatedRelativePath);
-
-                                    if (allPossiblePaths.has(calculatedNormalizedPath)) {
-                                        architectureMap.edges.push({
-                                            source: normalize(node.id),
-                                            target: calculatedNormalizedPath,
-                                            type: imp.type || 'import'
-                                        });
-                                        resolvedImports++;
-                                        targetFound = true;
-                                    }
-                                } catch (e) {
-                                    console.warn(`⚠️ Path calculation failed for import '${imp.source}' in file '${node.id}': ${e.message}`);
-                                    // If path calculation fails, continue to unresolved
-                                }
-                            }
-
-                            if (!targetFound) {
-                                // Another fallback: if import starts with ./ or ../, try to find a match
-                                // by appending common extensions to the import path
-                                if (imp.source.startsWith('./') || imp.source.startsWith('../')) {
-                                    // Try various extensions that might match
-                                    const extensions = ['', '.js', '.ts', '.jsx', '.tsx', '.mjs', '.cjs'];
-
-                                    for (const ext of extensions) {
-                                        let testPath = imp.source;
-                                        if (!imp.source.endsWith(ext)) {
-                                            testPath = imp.source + ext;
-                                        }
-
-                                        // Calculate the relative path from the project directory
-                                        try {
-                                            const baseDir = path.dirname(path.join(directory, node.id));
-                                            const calculatedAbsolutePath = path.resolve(baseDir, testPath);
-                                            const calculatedRelativePath = path.relative(directory, calculatedAbsolutePath);
-                                            const calculatedNormalizedPath = normalize(calculatedRelativePath);
-
-                                            if (allPossiblePaths.has(calculatedNormalizedPath)) {
-                                                architectureMap.edges.push({
-                                                    source: normalize(node.id),
-                                                    target: calculatedNormalizedPath,
-                                                    type: imp.type || 'import'
-                                                });
-                                                resolvedImports++;
-                                                targetFound = true;
-                                                break; // Found a match, exit the loop
-                                            }
-                                        } catch (e) {
-                                            console.warn(`⚠️ Extension path calculation failed for import '${imp.source}' with extension '${ext}' in file '${node.id}': ${e.message}`);
-                                            // Continue to next extension if path calculation fails
-                                        }
-                                    }
-                                }
-                            }
-
-                            if (!targetFound) {
-                                // Additional fallback: Check for index files when importing a directory
-                                if (imp.source.startsWith('./') || imp.source.startsWith('../')) {
-                                    try {
-                                        const baseDir = path.dirname(path.join(directory, node.id));
-                                        const calculatedAbsolutePath = path.resolve(baseDir, imp.source);
-                                        const directoryPath = calculatedAbsolutePath;
-
-                                        // Check for index files in the directory
-                                        const indexFiles = ['index.js', 'index.ts', 'index.jsx', 'index.tsx', 'index.mjs', 'index.cjs'];
-                                        for (const indexFile of indexFiles) {
-                                            const indexPath = path.join(directoryPath, indexFile);
-                                            const indexRelativePath = path.relative(directory, indexPath);
-                                            const indexNormalizedPath = normalize(indexRelativePath);
-
-                                            if (allPossiblePaths.has(indexNormalizedPath)) {
-                                                architectureMap.edges.push({
-                                                    source: normalize(node.id),
-                                                    target: indexNormalizedPath,
-                                                    type: imp.type || 'import'
-                                                });
-                                                resolvedImports++;
-                                                targetFound = true;
-                                                break; // Found a match, exit the loop
-                                            }
-                                        }
-                                    } catch (e) {
-                                        console.warn(`⚠️ Index file check failed for import '${imp.source}' in file '${node.id}': ${e.message}`);
-                                        // Continue if index file check fails
-                                    }
-                                }
-                            }
-
-                            if (!targetFound) {
-                                // Additional fallback: Check for absolute imports that might map to common directories
-                                // For example, if tsconfig has paths like "@/*": ["src/*"], an import like "@/utils/helper" 
-                                // should resolve to "src/utils/helper.js" or similar
-                                if (imp.source.startsWith('@') || imp.source.startsWith('/') || !imp.source.startsWith('.')) {
-                                    // Try to match against common source directories
-                                    const commonSourceDirs = ['src', 'app', 'lib', 'components', 'utils', 'services', 'assets'];
-
-                                    for (const srcDir of commonSourceDirs) {
-                                        // Try appending to common source directories
-                                        const potentialPath = path.join(srcDir, imp.source);
-                                        const potentialRelativePath = path.relative(directory, path.resolve(directory, potentialPath));
-                                        const potentialNormalizedPath = normalize(potentialRelativePath);
-
-                                        if (allPossiblePaths.has(potentialNormalizedPath)) {
-                                            architectureMap.edges.push({
-                                                source: normalize(node.id),
-                                                target: potentialNormalizedPath,
-                                                type: imp.type || 'import'
-                                            });
-                                            resolvedImports++;
-                                            targetFound = true;
-                                            break; // Found a match, exit the loop
-                                        }
-
-                                        // Also try with extensions
-                                        const extensions = ['.js', '.ts', '.jsx', '.tsx', '.mjs', '.cjs',
-                                            'index.js', 'index.ts', 'index.jsx', 'index.tsx', 'index.mjs', 'index.cjs'];
-
-                                        for (const ext of extensions) {
-                                            let testPath;
-                                            if (ext.startsWith('index')) {
-                                                testPath = path.join(srcDir, imp.source, ext);
-                                            } else {
-                                                testPath = path.join(srcDir, imp.source + ext);
-                                            }
-
-                                            const testRelativePath = path.relative(directory, path.resolve(directory, testPath));
-                                            const testNormalizedPath = normalize(testRelativePath);
-
-                                            if (allPossiblePaths.has(testNormalizedPath)) {
-                                                architectureMap.edges.push({
-                                                    source: normalize(node.id),
-                                                    target: testNormalizedPath,
-                                                    type: imp.type || 'import'
-                                                });
-                                                resolvedImports++;
-                                                targetFound = true;
-                                                break; // Found a match, exit the loops
-                                            }
-                                        }
-
-                                        if (targetFound) break;
-                                    }
-                                }
-                            }
-
-                            if (!targetFound) {
-                                // Additional fallback: Try to handle barrel files (index.js that exports from other files)
-                                // If import is '@/components/Header' but only '@/components/index.js' exists
-                                if (imp.source.startsWith('@') || imp.source.startsWith('/')) {
-                                    try {
-                                        // Try to find a directory with an index file that matches
-                                        const parts = imp.source.split('/');
-                                        if (parts.length > 1) {
-                                            // Remove the last part and add index
-                                            const dirParts = parts.slice(0, -1);
-                                            const fileName = parts[parts.length - 1];
-
-                                            // Look for patterns like: components/index.js exporting { Header }
-                                            // or components/index.js containing export * from './Header'
-                                            const dirPath = dirParts.join('/') + '/index';
-                                            const indexFiles = ['index.js', 'index.ts', 'index.jsx', 'index.tsx', 'index.mjs', 'index.cjs'];
-
-                                            for (const indexFile of indexFiles) {
-                                                const indexPath = path.join(dirParts.join('/'), indexFile);
-                                                const indexRelativePath = path.relative(directory, path.resolve(directory, indexPath));
-                                                const indexNormalizedPath = normalize(indexRelativePath);
-
-                                                if (allPossiblePaths.has(indexNormalizedPath)) {
-                                                    architectureMap.edges.push({
-                                                        source: normalize(node.id),
-                                                        target: indexNormalizedPath,
-                                                        type: imp.type || 'import'
-                                                    });
-                                                    resolvedImports++;
-                                                    targetFound = true;
-                                                    break; // Found a match, exit the loop
-                                                }
-                                            }
-                                        }
-                                    } catch (e) {
-                                        console.warn(`⚠️ Barrel file check failed for import '${imp.source}' in file '${node.id}': ${e.message}`);
-                                        // Continue if barrel file check fails
-                                    }
-                                }
-                            }
-
-                            if (!targetFound) {
-                                // Additional fallback: Check for exact path matches with different capitalization
-                                // This handles cases where import uses different casing than actual file
-                                for (const possiblePath of allPossiblePaths) {
-                                    if (possiblePath.toLowerCase() === imp.source.toLowerCase()) {
-                                        architectureMap.edges.push({
-                                            source: normalize(node.id),
-                                            target: possiblePath,
-                                            type: imp.type || 'import'
-                                        });
-                                        resolvedImports++;
-                                        targetFound = true;
-                                        break; // Found a case-insensitive match
-                                    }
-                                }
-                            }
-                        } catch (e) {
-                            console.error(`❌ Critical error processing import '${imp.source}' in file '${node.id}': ${e.message}`);
-                            console.error(e.stack);
-                            // Even if there's an error processing this import, increment unresolved counter
-                            unresolvedImports++;
-                        }
-
-                        if (!targetFound) {
-                            // Track imports that couldn't be matched to existing nodes
-                            unresolvedImports++;
-                        }
-                    }
-                });
-            }
-        });
-        console.log('RESOLVED IMPORTS:', resolvedImports);
-        console.log('UNRESOLVED IMPORTS:', unresolvedImports);
-        console.log('IMPORT EDGES CREATED:', architectureMap.edges.length);
-
-        // NEW: Add semantic usage edges
-        if (semanticResults && semanticResults.usageEdges) {
-            semanticResults.usageEdges.forEach(usageEdge => {
-                // Only add if both source and target exist in our nodes
-                const sourceExists = architectureMap.nodes.some(n => n.id === usageEdge.source);
-                const targetExists = architectureMap.nodes.some(n => n.id === usageEdge.target);
-
-                if (sourceExists && targetExists) {
-                    architectureMap.edges.push({
-                        source: normalize(usageEdge.source),
-                        target: normalize(usageEdge.target),
-                        type: usageEdge.type // 'function_call', 'constructor_call', 'component_usage'
-                    });
-                }
-            });
-        }
-        console.log('TOTAL EDGES (with usage):', architectureMap.edges.length);
-
-        // Calculate blast radius for each file
-        const reverseGraph = buildReverseDependencyGraph(architectureMap);
-        const blastRadiusMap = computeBlastRadius(reverseGraph);
-
-        // Add blast_radius to each node
-        architectureMap.nodes.forEach(node => {
-            node.metadata.blast_radius = blastRadiusMap[node.id] || 0;
-        });
-
-        console.log('BEFORE DEDUPLICATION EDGES:', architectureMap.edges.length);
-
-        // Process edges to ensure they match existing nodes and remove duplicates
-        const validEdges = [];
-        const edgeSet = new Set(); // To track unique edges
-
-        architectureMap.edges.forEach(edge => {
-            // Normalize both source and target paths for consistent matching
-            const normalizedSource = normalize(edge.source);
-            const normalizedTarget = normalize(edge.target);
-
-            // Create a unique key for the edge to avoid duplicates
-            const edgeKey = `${normalizedSource}→${normalizedTarget}`;
-
-            // Check if both source and target nodes exist in our architecture map
-            const sourceNodeExists = architectureMap.nodes.some(node => node.id === normalizedSource);
-            const targetNodeExists = architectureMap.nodes.some(node => node.id === normalizedTarget);
-
-            if (sourceNodeExists && targetNodeExists && !edgeSet.has(edgeKey)) {
-                edgeSet.add(edgeKey);
-                validEdges.push({
-                    source: normalizedSource,
-                    target: normalizedTarget,
-                    type: edge.type
-                });
-            }
-        });
-
-        console.log('AFTER DEDUPLICATION EDGES:', validEdges.length);
-
-        // Replace the edges with only valid ones
-        architectureMap.edges = validEdges;
-
-        // Add top blast radius files to the architecture map
-        const totalFiles = architectureMap.nodes.length;
-        const { computeBlastRadiusWithPercentage, analyzeCriticality } = require('./blastRadius');
-        const allFilesWithPercentage = computeBlastRadiusWithPercentage(blastRadiusMap, totalFiles);
-
-        // Analyze criticality of files
-        const criticalityAnalysis = analyzeCriticality(blastRadiusMap, reverseGraph, architectureMap.nodes);
-
-        // Sort by blast radius descending and take top 3
-        const topFiles = allFilesWithPercentage.slice(0, 3);
-
-        architectureMap.contextSurface = {
-            topBlastRadiusFiles: topFiles,
-            criticalityAnalysis: criticalityAnalysis.slice(0, 5) // Top 5 most critical files
-        };
-
-        // Build the new context object that conforms to the schema
-        let context = buildContext(architectureMap.nodes, architectureMap.edges, {
-            directory: directory,
-            projectName: path.basename(directory),
-            language: 'javascript',
-            contextSurface: architectureMap.contextSurface
-        });
-
-        console.log('Validating structural context...');
-
-        // Validate the context against the schema
-        let validation = validateContext(context);
-
-        if (!validation.valid) {
-            console.error('VALIDATION FAILED (Attempt 1)');
-            validation.errors.forEach(e => console.error(' -', e));
-
-            console.log('Re-running generation deterministically...');
-
-            // Rebuild context deterministically
-            context = buildContext(architectureMap.nodes, architectureMap.edges, {
-                directory: directory,
-                projectName: path.basename(directory),
-                language: 'javascript',
-                contextSurface: architectureMap.contextSurface
-            });
-
-            validation = validateContext(context);
-
-            if (!validation.valid) {
-                console.error('VALIDATION FAILED (Attempt 2)');
-                validation.errors.forEach(e => console.error(' -', e));
-                console.error('ABORTING. JSON NOT SAVED.');
-                process.exit(1);
-            }
-        }
-
-        // Sort the context deterministically
-        const sortedContext = sortContext(context);
-
-        console.log('STRUCTURAL CONTEXT VALIDATED — READY FOR UPLOAD');
-
-        return sortedContext;
-
-    } catch (err) {
-        throw err;
-    }
-}
-
-module.exports = { scan };