archicore 0.3.5 → 0.3.7

package/dist/server/services/enterprise-indexer.d.ts

@@ -0,0 +1,116 @@
+/**
+ * Enterprise Indexer
+ *
+ * Handles large-scale projects (50K+ files) with intelligent sampling,
+ * incremental indexing, and tiered analysis modes.
+ */
+/**
+ * Enterprise indexing options
+ */
+export interface EnterpriseIndexingOptions {
+    tier: 'quick' | 'standard' | 'deep';
+    sampling: {
+        enabled: boolean;
+        maxFiles: number;
+        strategy: 'hot-files' | 'random' | 'directory-balanced' | 'smart';
+    };
+    incremental: {
+        enabled: boolean;
+        since?: string;
+    };
+    focusDirectories?: string[];
+    excludePatterns?: string[];
+    memoryLimitMB?: number;
+}
+/**
+ * Analysis tier configuration
+ */
+declare const TIER_CONFIG: {
+    quick: {
+        maxFiles: number;
+        skipEmbeddings: boolean;
+        skipSecurity: boolean;
+        skipDuplication: boolean;
+        description: string;
+    };
+    standard: {
+        maxFiles: number;
+        skipEmbeddings: boolean;
+        skipSecurity: boolean;
+        skipDuplication: boolean;
+        description: string;
+    };
+    deep: {
+        maxFiles: number;
+        skipEmbeddings: boolean;
+        skipSecurity: boolean;
+        skipDuplication: boolean;
+        description: string;
+    };
+};
+/**
+ * Default options for enterprise indexing
+ */
+export declare const DEFAULT_ENTERPRISE_OPTIONS: EnterpriseIndexingOptions;
+export declare class EnterpriseIndexer {
+    private projectPath;
+    private options;
+    private gitAvailable;
+    constructor(projectPath: string, options?: Partial<EnterpriseIndexingOptions>);
+    /**
+     * Initialize - check git availability
+     */
+    initialize(): Promise<void>;
+    /**
+     * Get project size estimation
+     */
+    getProjectSize(): Promise<{
+        totalFiles: number;
+        totalSizeMB: number;
+        languageDistribution: Record<string, number>;
+        recommendation: 'quick' | 'standard' | 'deep';
+        estimatedTimeMinutes: number;
+    }>;
+    /**
+     * Get files to index based on sampling strategy
+     */
+    getFilesToIndex(): Promise<string[]>;
+    /**
+     * Get files changed since last index (for incremental indexing)
+     */
+    getChangedFiles(since: string): Promise<string[]>;
+    /**
+     * Scan all files in project
+     */
+    private scanAllFiles;
+    /**
+     * Select hot files (most frequently changed)
+     */
+    private selectHotFiles;
+    /**
+     * Random selection
+     */
+    private selectRandom;
+    /**
+     * Directory-balanced selection
+     */
+    private selectDirectoryBalanced;
+    /**
+     * Smart selection - combines multiple heuristics
+     */
+    private selectSmart;
+    /**
+     * Get import counts for files
+     */
+    private getImportCounts;
+    /**
+     * Get commit counts from git
+     */
+    private getCommitCounts;
+    /**
+     * Get tier configuration
+     */
+    getTierConfig(): typeof TIER_CONFIG[keyof typeof TIER_CONFIG];
+}
+export { TIER_CONFIG };
+//# sourceMappingURL=enterprise-indexer.d.ts.map

package/dist/server/services/enterprise-indexer.js

@@ -0,0 +1,529 @@
+/**
+ * Enterprise Indexer
+ *
+ * Handles large-scale projects (50K+ files) with intelligent sampling,
+ * incremental indexing, and tiered analysis modes.
+ */
+import { exec } from 'child_process';
+import { promisify } from 'util';
+import { existsSync, statSync } from 'fs';
+import { readdir, stat, readFile } from 'fs/promises';
+import path from 'path';
+import { Logger } from '../../utils/logger.js';
+const execAsync = promisify(exec);
+/**
+ * Analysis tier configuration
+ */
+const TIER_CONFIG = {
+    quick: {
+        maxFiles: 1000,
+        skipEmbeddings: true,
+        skipSecurity: true,
+        skipDuplication: true,
+        description: 'Fast overview - structure and basic metrics only'
+    },
+    standard: {
+        maxFiles: 5000,
+        skipEmbeddings: false,
+        skipSecurity: false,
+        skipDuplication: false,
+        description: 'Standard analysis - all features with sampling'
+    },
+    deep: {
+        maxFiles: 50000,
+        skipEmbeddings: false,
+        skipSecurity: false,
+        skipDuplication: false,
+        description: 'Deep analysis - comprehensive analysis for entire project'
+    }
+};
+/**
+ * Default options for enterprise indexing
+ */
+export const DEFAULT_ENTERPRISE_OPTIONS = {
+    tier: 'standard',
+    sampling: {
+        enabled: true,
+        maxFiles: 5000,
+        strategy: 'smart'
+    },
+    incremental: {
+        enabled: false
+    },
+    excludePatterns: [
+        'node_modules',
+        '.git',
+        'dist',
+        'build',
+        '__pycache__',
+        '.cache',
+        'vendor',
+        'coverage',
+        '.next',
+        '.nuxt'
+    ],
+    memoryLimitMB: 2048
+};
+export class EnterpriseIndexer {
+    projectPath;
+    options;
+    gitAvailable = false;
+    constructor(projectPath, options = {}) {
+        this.projectPath = projectPath;
+        this.options = { ...DEFAULT_ENTERPRISE_OPTIONS, ...options };
+        // Apply tier config
+        const tierConfig = TIER_CONFIG[this.options.tier];
+        if (this.options.sampling.enabled && !options.sampling?.maxFiles) {
+            this.options.sampling.maxFiles = tierConfig.maxFiles;
+        }
+    }
+    /**
+     * Initialize - check git availability
+     */
+    async initialize() {
+        try {
+            await execAsync('git --version', { cwd: this.projectPath });
+            const gitDir = path.join(this.projectPath, '.git');
+            this.gitAvailable = existsSync(gitDir);
+        }
+        catch {
+            this.gitAvailable = false;
+        }
+        Logger.info(`Enterprise indexer initialized. Git available: ${this.gitAvailable}`);
+    }
+    /**
+     * Get project size estimation
+     */
+    async getProjectSize() {
+        const files = await this.scanAllFiles();
+        const totalSize = files.reduce((sum, f) => sum + (f.size || 0), 0);
+        // Count by language
+        const languageDistribution = {};
+        for (const file of files) {
+            const ext = path.extname(file.path).toLowerCase() || '.other';
+            languageDistribution[ext] = (languageDistribution[ext] || 0) + 1;
+        }
+        // Recommendation based on size
+        let recommendation;
+        if (files.length < 1000) {
+            recommendation = 'deep';
+        }
+        else if (files.length < 10000) {
+            recommendation = 'standard';
+        }
+        else {
+            recommendation = 'quick';
+        }
+        // Estimate time (rough)
+        // Quick: ~10 files/sec, Standard: ~5 files/sec, Deep: ~2 files/sec
+        const filesPerSecond = recommendation === 'quick' ? 10 : recommendation === 'standard' ? 5 : 2;
+        const filesToProcess = Math.min(files.length, TIER_CONFIG[recommendation].maxFiles);
+        const estimatedTimeMinutes = Math.ceil(filesToProcess / filesPerSecond / 60);
+        return {
+            totalFiles: files.length,
+            totalSizeMB: Math.round(totalSize / (1024 * 1024) * 100) / 100,
+            languageDistribution,
+            recommendation,
+            estimatedTimeMinutes
+        };
+    }
+    /**
+     * Get files to index based on sampling strategy
+     */
+    async getFilesToIndex() {
+        const allFiles = await this.scanAllFiles();
+        Logger.info(`Total files found: ${allFiles.length}`);
+        if (!this.options.sampling.enabled) {
+            return allFiles.map(f => f.path);
+        }
+        const maxFiles = this.options.sampling.maxFiles;
+        if (allFiles.length <= maxFiles) {
+            Logger.info(`All files within limit (${allFiles.length} <= ${maxFiles})`);
+            return allFiles.map(f => f.path);
+        }
+        // Apply sampling strategy
+        let selectedFiles;
+        switch (this.options.sampling.strategy) {
+            case 'hot-files':
+                selectedFiles = await this.selectHotFiles(allFiles, maxFiles);
+                break;
+            case 'random':
+                selectedFiles = this.selectRandom(allFiles, maxFiles);
+                break;
+            case 'directory-balanced':
+                selectedFiles = this.selectDirectoryBalanced(allFiles, maxFiles);
+                break;
+            case 'smart':
+            default:
+                selectedFiles = await this.selectSmart(allFiles, maxFiles);
+        }
+        Logger.info(`Selected ${selectedFiles.length} files using '${this.options.sampling.strategy}' strategy`);
+        return selectedFiles;
+    }
+    /**
+     * Get files changed since last index (for incremental indexing)
+     */
+    async getChangedFiles(since) {
+        if (!this.gitAvailable) {
+            Logger.warn('Git not available - cannot determine changed files');
+            return [];
+        }
+        try {
+            // Try to use git to get changed files
+            let command;
+            if (since.match(/^[0-9a-f]{7,40}$/i)) {
+                // Git commit hash
+                command = `git diff --name-only ${since}..HEAD`;
+            }
+            else {
+                // ISO date
+                command = `git log --since="${since}" --name-only --pretty=format: | sort -u`;
+            }
+            const { stdout } = await execAsync(command, { cwd: this.projectPath });
+            const changedFiles = stdout.split('\n')
+                .map(f => f.trim())
+                .filter(f => f.length > 0)
+                .map(f => path.join(this.projectPath, f))
+                .filter(f => existsSync(f));
+            Logger.info(`Found ${changedFiles.length} changed files since ${since}`);
+            return changedFiles;
+        }
+        catch (error) {
+            Logger.error('Failed to get changed files:', error);
+            return [];
+        }
+    }
+    /**
+     * Scan all files in project
+     */
+    async scanAllFiles() {
+        const files = [];
+        const excludePatterns = this.options.excludePatterns || [];
+        const CODE_EXTENSIONS = new Set([
+            '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs',
+            '.py', '.rb', '.go', '.rs', '.java', '.kt', '.scala',
+            '.c', '.cpp', '.h', '.hpp', '.cs', '.swift',
+            '.vue', '.svelte', '.astro',
+            '.php', '.pl', '.pm', '.lua', '.r',
+            '.sql', '.graphql', '.gql',
+            '.yaml', '.yml', '.json', '.toml', '.xml'
+        ]);
+        const scanDir = async (dir) => {
+            try {
+                const entries = await readdir(dir, { withFileTypes: true });
+                for (const entry of entries) {
+                    const fullPath = path.join(dir, entry.name);
+                    // Check exclude patterns
+                    if (excludePatterns.some(p => fullPath.includes(p))) {
+                        continue;
+                    }
+                    if (entry.isDirectory()) {
+                        await scanDir(fullPath);
+                    }
+                    else if (entry.isFile()) {
+                        const ext = path.extname(entry.name).toLowerCase();
+                        if (CODE_EXTENSIONS.has(ext)) {
+                            try {
+                                const stats = await stat(fullPath);
+                                // Skip very large files (> 1MB)
+                                if (stats.size < 1024 * 1024) {
+                                    files.push({ path: fullPath, size: stats.size });
+                                }
+                            }
+                            catch {
+                                // Skip files we can't stat
+                            }
+                        }
+                    }
+                }
+            }
+            catch {
+                // Skip directories we can't read
+            }
+        };
+        await scanDir(this.projectPath);
+        return files;
+    }
+    /**
+     * Select hot files (most frequently changed)
+     */
+    async selectHotFiles(files, maxFiles) {
+        if (!this.gitAvailable) {
+            Logger.warn('Git not available, falling back to smart selection');
+            return this.selectSmart(files, maxFiles);
+        }
+        const fileScores = [];
+        // Get commit counts for each file
+        try {
+            const { stdout } = await execAsync('git log --name-only --pretty=format: | sort | uniq -c | sort -rn | head -5000', { cwd: this.projectPath, maxBuffer: 10 * 1024 * 1024 });
+            const commitCounts = new Map();
+            for (const line of stdout.split('\n')) {
+                const match = line.trim().match(/^\s*(\d+)\s+(.+)$/);
+                if (match) {
+                    const count = parseInt(match[1], 10);
+                    const filePath = path.join(this.projectPath, match[2]);
+                    commitCounts.set(filePath, count);
+                }
+            }
+            for (const file of files) {
+                const commitCount = commitCounts.get(file.path) || 0;
+                fileScores.push({
+                    path: file.path,
+                    score: commitCount,
+                    reasons: [`${commitCount} commits`],
+                    lastModified: new Date(),
+                    commitCount
+                });
+            }
+        }
+        catch {
+            // Fallback to modification time
+            for (const file of files) {
+                try {
+                    const stats = statSync(file.path);
+                    fileScores.push({
+                        path: file.path,
+                        score: stats.mtimeMs,
+                        reasons: ['recent modification'],
+                        lastModified: stats.mtime
+                    });
+                }
+                catch {
+                    fileScores.push({
+                        path: file.path,
+                        score: 0,
+                        reasons: [],
+                        lastModified: new Date(0)
+                    });
+                }
+            }
+        }
+        // Sort by score and select top N
+        fileScores.sort((a, b) => b.score - a.score);
+        return fileScores.slice(0, maxFiles).map(f => f.path);
+    }
+    /**
+     * Random selection
+     */
+    selectRandom(files, maxFiles) {
+        const shuffled = [...files].sort(() => Math.random() - 0.5);
+        return shuffled.slice(0, maxFiles).map(f => f.path);
+    }
+    /**
+     * Directory-balanced selection
+     */
+    selectDirectoryBalanced(files, maxFiles) {
+        // Group by top-level directory
+        const byDirectory = new Map();
+        for (const file of files) {
+            const relativePath = path.relative(this.projectPath, file.path);
+            const topDir = relativePath.split(path.sep)[0] || '.';
+            if (!byDirectory.has(topDir)) {
+                byDirectory.set(topDir, []);
+            }
+            byDirectory.get(topDir).push(file);
+        }
+        // Calculate files per directory
+        const dirCount = byDirectory.size;
+        const filesPerDir = Math.ceil(maxFiles / dirCount);
+        const selected = [];
+        for (const [, dirFiles] of byDirectory) {
+            // Sort by recency (assume larger files are more important)
+            dirFiles.sort((a, b) => b.size - a.size);
+            const toTake = Math.min(filesPerDir, dirFiles.length);
+            selected.push(...dirFiles.slice(0, toTake).map(f => f.path));
+        }
+        // If we have room, add more from the largest directories
+        if (selected.length < maxFiles) {
+            const remaining = maxFiles - selected.length;
+            const allRemaining = files
+                .filter(f => !selected.includes(f.path))
+                .sort((a, b) => b.size - a.size)
+                .slice(0, remaining);
+            selected.push(...allRemaining.map(f => f.path));
+        }
+        return selected.slice(0, maxFiles);
+    }
+    /**
+     * Smart selection - combines multiple heuristics
+     */
+    async selectSmart(files, maxFiles) {
+        const fileScores = [];
+        // Priority patterns (entry points, configs, core logic)
+        const HIGH_PRIORITY_PATTERNS = [
+            /index\.(ts|js|tsx|jsx)$/,
+            /main\.(ts|js|py|go|rs)$/,
+            /app\.(ts|js|tsx|jsx|vue)$/,
+            /server\.(ts|js)$/,
+            /config\.(ts|js|json|yaml|yml)$/,
+            /routes?\.(ts|js)$/,
+            /api\.(ts|js)$/,
+            /service\.(ts|js)$/,
+            /controller\.(ts|js)$/,
+            /model\.(ts|js|py)$/,
+            /schema\.(ts|js|graphql)$/,
+            /types?\.(ts|d\.ts)$/,
+            /package\.json$/,
+            /tsconfig\.json$/,
+            /\.env\.example$/
+        ];
+        // Get import counts if possible
+        const importCounts = await this.getImportCounts(files);
+        // Get commit counts if git available
+        let commitCounts = new Map();
+        if (this.gitAvailable) {
+            commitCounts = await this.getCommitCounts();
+        }
+        for (const file of files) {
+            const relativePath = path.relative(this.projectPath, file.path);
+            let score = 0;
+            const reasons = [];
+            // Check priority patterns
+            for (const pattern of HIGH_PRIORITY_PATTERNS) {
+                if (pattern.test(relativePath)) {
+                    score += 100;
+                    reasons.push('priority file');
+                    break;
+                }
+            }
+            // Focus directories get higher priority
+            if (this.options.focusDirectories) {
+                for (const focusDir of this.options.focusDirectories) {
+                    if (relativePath.startsWith(focusDir)) {
+                        score += 50;
+                        reasons.push('focus directory');
+                        break;
+                    }
+                }
+            }
+            // Bonus for shallow depth (closer to root = more important)
+            const depth = relativePath.split(path.sep).length;
+            score += Math.max(0, 20 - depth * 2);
+            // Bonus for import count
+            const imports = importCounts.get(file.path) || 0;
+            if (imports > 0) {
+                score += Math.min(imports * 5, 50);
+                reasons.push(`${imports} imports`);
+            }
+            // Bonus for commit count
+            const commits = commitCounts.get(file.path) || 0;
+            if (commits > 0) {
+                score += Math.min(commits * 2, 30);
+                reasons.push(`${commits} commits`);
+            }
+            // Penalty for test files (still include some)
+            if (relativePath.includes('test') || relativePath.includes('spec') || relativePath.includes('__tests__')) {
+                score -= 30;
+            }
+            // Penalty for generated files
+            if (relativePath.includes('.generated') || relativePath.includes('.d.ts')) {
+                score -= 40;
+            }
+            fileScores.push({
+                path: file.path,
+                score,
+                reasons,
+                lastModified: new Date(),
+                commitCount: commits,
+                importCount: imports
+            });
+        }
+        // Sort by score
+        fileScores.sort((a, b) => b.score - a.score);
+        // Log top 10 for debugging
+        Logger.debug('Top 10 files by score:');
+        for (const f of fileScores.slice(0, 10)) {
+            Logger.debug(`  ${f.score}: ${path.relative(this.projectPath, f.path)} (${f.reasons.join(', ')})`);
+        }
+        return fileScores.slice(0, maxFiles).map(f => f.path);
+    }
+    /**
+     * Get import counts for files
+     */
+    async getImportCounts(files) {
+        const importCounts = new Map();
+        // Quick scan for imports (simplified)
+        const importPatterns = [
+            /from\s+['"]([^'"]+)['"]/g,
+            /import\s+['"]([^'"]+)['"]/g,
+            /require\(['"]([^'"]+)['"]\)/g
+        ];
+        // Normalize import path to file path
+        const normalizeImport = (importPath, fromFile) => {
+            // Skip external packages
+            if (!importPath.startsWith('.') && !importPath.startsWith('/')) {
+                return null;
+            }
+            const fromDir = path.dirname(fromFile);
+            let resolved = path.resolve(fromDir, importPath);
+            // Try common extensions
+            const extensions = ['.ts', '.tsx', '.js', '.jsx', '.json', ''];
+            for (const ext of extensions) {
+                const withExt = resolved + ext;
+                if (existsSync(withExt)) {
+                    return withExt;
+                }
+                // Try index files
+                const indexPath = path.join(resolved, 'index' + ext);
+                if (existsSync(indexPath)) {
+                    return indexPath;
+                }
+            }
+            return null;
+        };
+        // Sample files for import counting (don't scan all)
+        const sampleSize = Math.min(1000, files.length);
+        const sample = files.slice(0, sampleSize);
+        for (const file of sample) {
+            try {
+                const content = await readFile(file.path, 'utf-8');
+                for (const pattern of importPatterns) {
+                    let match;
+                    while ((match = pattern.exec(content)) !== null) {
+                        const importPath = match[1];
+                        const resolvedPath = normalizeImport(importPath, file.path);
+                        if (resolvedPath) {
+                            importCounts.set(resolvedPath, (importCounts.get(resolvedPath) || 0) + 1);
+                        }
+                    }
+                }
+            }
+            catch {
+                // Skip files we can't read
+            }
+        }
+        return importCounts;
+    }
+    /**
+     * Get commit counts from git
+     */
+    async getCommitCounts() {
+        const commitCounts = new Map();
+        if (!this.gitAvailable) {
+            return commitCounts;
+        }
+        try {
+            const { stdout } = await execAsync('git log --name-only --pretty=format: --since="1 year ago" | sort | uniq -c | sort -rn | head -2000', { cwd: this.projectPath, maxBuffer: 10 * 1024 * 1024 });
+            for (const line of stdout.split('\n')) {
+                const match = line.trim().match(/^\s*(\d+)\s+(.+)$/);
+                if (match) {
+                    const count = parseInt(match[1], 10);
+                    const filePath = path.join(this.projectPath, match[2]);
+                    commitCounts.set(filePath, count);
+                }
+            }
+        }
+        catch {
+            // Ignore errors
+        }
+        return commitCounts;
+    }
+    /**
+     * Get tier configuration
+     */
+    getTierConfig() {
+        return TIER_CONFIG[this.options.tier];
+    }
+}
+export { TIER_CONFIG };
+//# sourceMappingURL=enterprise-indexer.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archicore",
-  "version": "0.3.5",
+  "version": "0.3.7",
   "description": "AI Software Architect - code analysis, impact prediction, semantic search",
   "main": "dist/index.js",
   "type": "module",
@@ -80,6 +80,7 @@
     "passport-github2": "^0.1.12",
     "pg": "^8.13.0",
     "ora": "^8.1.1",
+    "tar": "^7.4.3",
     "tree-sitter": "^0.21.1",
     "tree-sitter-javascript": "^0.21.4",
     "tree-sitter-python": "^0.21.0",