codesummary 1.2.0 → 1.2.2

package/src/ragGenerator.js

@@ -1,1740 +0,0 @@
-import fs from 'fs-extra';
-import path from 'path';
-import crypto from 'crypto';
-import os from 'os';
-import { createReadStream } from 'fs';
-import ErrorHandler from './errorHandler.js';
-import RagConfigManager from './ragConfig.js';
-import { formatFileSize } from './utils.js';
-
-/**
- * Professional RAG Generator for CodeSummary
- * Generates streaming JSON output optimized for vector database ingestion
- * Follows deterministic, AI-free approach with efficient memory usage
- */
-export class RagGenerator {
-  constructor() {
-    // Global parameters
-    this.maxTokensPerChunk = 1000;
-    this.overlapTokens = 200;
-    this.maxWorkers = Math.min(8, os.cpus().length);
-    
-    // Extension to language mapping (deterministic)
-    this.extensionToLanguage = {
-      '.js': 'JavaScript', '.jsx': 'JavaScript', '.ts': 'TypeScript', '.tsx': 'TypeScript',
-      '.py': 'Python', '.java': 'Java', '.cs': 'C#', '.cpp': 'C++', '.c': 'C', '.h': 'C/C++',
-      '.html': 'HTML', '.xml': 'XML', '.css': 'CSS', '.scss': 'SCSS',
-      '.json': 'JSON', '.yaml': 'YAML', '.yml': 'YAML',
-      '.md': 'Markdown', '.txt': 'Text',
-      '.sh': 'Shell', '.bat': 'Batch'
-    };
-    
-    // Initialize handlers
-    this.handlers = this.initializeHandlers();
-    
-    // Statistics tracking
-    this.stats = {
-      filesProcessed: 0,
-      chunksGenerated: 0,
-      bytesWritten: 0,
-      startTime: null,
-      endTime: null
-    };
-    
-    
-    // Error collection
-    this.errors = [];
-  }
-
-  /**
-   * Main entry point - generates streaming RAG JSON
-   * @param {object} filesByExtension - Files grouped by extension
-   * @param {Array} selectedExtensions - Selected extensions to process
-   * @param {string} outputPath - Output JSON file path
-   * @param {string} projectName - Project name
-   * @param {string} scanPath - Root scan path
-   * @returns {object} Generation result
-   */
-  async generateRagOutput(filesByExtension, selectedExtensions, outputPath, projectName, scanPath) {
-    this.stats.startTime = Date.now();
-    
-    try {
-      console.log(`🚀 Starting RAG generation for ${projectName}`);
-      
-      // Load RAG configuration
-      this.ragConfigManager = new RagConfigManager();
-      const config = await this.ragConfigManager.loadConfig();
-      this.updateConfigFromYAML(config);
-
-      // Display configuration
-      this.ragConfigManager.displayConfig();
-      
-      // Phase 1: Discovery and file preparation
-      const discoveredFiles = await this.discoveryPhase(filesByExtension, selectedExtensions, scanPath);
-      
-      // Phase 2: Atomic JSON generation (thread-safe)
-      const result = await this.generate(discoveredFiles, outputPath, projectName, scanPath);
-      
-      this.stats.endTime = Date.now();
-      const duration = (this.stats.endTime - this.stats.startTime) / 1000;
-      
-      console.log(`✅ RAG generation completed in ${duration.toFixed(2)}s`);
-      console.log(`📊 Stats: ${this.stats.filesProcessed} files, ${this.stats.chunksGenerated} chunks`);
-      
-      return {
-        outputPath,
-        totalFiles: this.stats.filesProcessed,
-        totalChunks: this.stats.chunksGenerated,
-        duration,
-        success: true
-      };
-      
-    } catch (error) {
-      ErrorHandler.handleError(error, 'RAG Generation');
-      throw error;
-    }
-  }
-
-  /**
-   * Update internal configuration from loaded YAML config
-   * @param {object} yamlConfig - Configuration from YAML
-   */
-  updateConfigFromYAML(yamlConfig) {
-    if (yamlConfig.chunking) {
-      this.maxTokensPerChunk = yamlConfig.chunking.maxTokens || this.maxTokensPerChunk;
-      this.overlapTokens = yamlConfig.chunking.overlap || this.overlapTokens;
-    }
-    
-    if (yamlConfig.performance) {
-      this.maxWorkers = Math.min(
-        yamlConfig.performance.maxWorkers || this.maxWorkers,
-        os.cpus().length
-      );
-    }
-    
-    // Store full config for handlers to use
-    this.yamlConfig = yamlConfig;
-  }
-
-
-
-  /**
-   * Phase 1: Discovery - BFS traversal and file metadata collection
-   * @param {object} filesByExtension - Files by extension
-   * @param {Array} selectedExtensions - Selected extensions
-   * @param {string} scanPath - Root scan path
-   * @returns {Array} Discovered files with metadata
-   */
-  async discoveryPhase(filesByExtension, selectedExtensions, scanPath) {
-    console.log('🔍 Discovery phase: collecting file metadata...');
-    
-    const discoveredFiles = [];
-    let processed = 0;
-    const totalFiles = selectedExtensions.reduce((sum, ext) => sum + (filesByExtension[ext]?.length || 0), 0);
-    
-    // Process files concurrently but limit memory usage
-    const batchSize = 50;
-    
-    for (const extension of selectedExtensions) {
-      const files = filesByExtension[extension] || [];
-      
-      for (let i = 0; i < files.length; i += batchSize) {
-        const batch = files.slice(i, i + batchSize);
-        const batchResults = await Promise.all(
-          batch.map(fileInfo => this.enrichFileMetadata(fileInfo, extension, scanPath))
-        );
-        
-        discoveredFiles.push(...batchResults.filter(Boolean));
-        processed += batch.length;
-        
-        // Progress reporting with validation
-        const progress = (processed / totalFiles * 100).toFixed(1);
-        const validFiles = discoveredFiles.length;
-        const skippedFiles = processed - validFiles;
-        process.stdout.write(`\r📊 Discovery: ${progress}% (${validFiles} valid, ${skippedFiles} skipped)`);
-        
-        // Internal validation
-        if (processed % 50 === 0) {
-          this.validateDiscoveryProgress(discoveredFiles, processed);
-        }
-      }
-    }
-    
-    console.log(`\n✅ Discovery completed: ${discoveredFiles.length} files enriched`);
-    return discoveredFiles;
-  }
-
-  /**
-   * Enrich file with metadata including hash, tags, and analysis
-   * @param {object} fileInfo - Basic file info from scanner
-   * @param {string} extension - File extension
-   * @param {string} scanPath - Root scan path
-   * @returns {object} Enriched file metadata
-   */
-  async enrichFileMetadata(fileInfo, extension, scanPath) {
-    try {
-      // Calculate SHA-256 hash in streaming mode
-      const hash = await this.calculateFileHash(fileInfo.absolutePath);
-      
-      // Determine language and tags
-      const language = this.extensionToLanguage[extension] || 'Unknown';
-      const tags = this.extractFileTags(fileInfo.relativePath, extension);
-      
-      // Basic file stats
-      const stats = await fs.stat(fileInfo.absolutePath);
-      
-      return {
-        id: hash.substring(0, 16), // Use first 16 chars of hash as unique ID
-        path: fileInfo.relativePath,
-        absolutePath: fileInfo.absolutePath,
-        extension,
-        language,
-        size: stats.size,
-        hash: `sha256-${hash}`,
-        modified: stats.mtime.toISOString(),
-        tags,
-        // Will be populated during chunking
-        chunks: null,
-        // Metadata for processing
-        _stats: stats
-      };
-      
-    } catch (error) {
-      console.warn(`⚠️  Could not process file ${fileInfo.relativePath}: ${error.message}`);
-      return null;
-    }
-  }
-
-  /**
-   * Calculate SHA-256 hash of file in streaming mode
-   * @param {string} filePath - File path
-   * @returns {string} SHA-256 hash (hex)
-   */
-  async calculateFileHash(filePath) {
-    return new Promise((resolve, reject) => {
-      const hash = crypto.createHash('sha256');
-      const stream = createReadStream(filePath);
-      
-      stream.on('data', data => hash.update(data));
-      stream.on('end', () => resolve(hash.digest('hex')));
-      stream.on('error', reject);
-    });
-  }
-
-  /**
-   * Extract file tags based on path heuristics
-   * @param {string} relativePath - Relative file path
-   * @param {string} extension - File extension
-   * @returns {Array} Array of tags
-   */
-  extractFileTags(relativePath, extension) {
-    const tags = [];
-    const pathLower = relativePath.toLowerCase();
-    const fileName = path.basename(relativePath, extension).toLowerCase();
-    const fullPath = relativePath.toLowerCase();
-    
-    // Path-based tags (enhanced)
-    if (pathLower.includes('/test/') || pathLower.includes('\\test\\')) tags.push('test');
-    if (pathLower.includes('/spec/') || pathLower.includes('\\spec\\')) tags.push('test');
-    if (pathLower.includes('/__tests__/') || pathLower.includes('\\__tests__\\')) tags.push('test');
-    if (pathLower.includes('/scripts/') || pathLower.includes('\\scripts\\')) tags.push('script');
-    if (pathLower.includes('/config/') || pathLower.includes('\\config\\')) tags.push('config');
-    if (pathLower.includes('/lib/') || pathLower.includes('\\lib\\')) tags.push('library');
-    if (pathLower.includes('/utils/') || pathLower.includes('\\utils\\')) tags.push('utility');
-    if (pathLower.includes('/helpers/') || pathLower.includes('\\helpers\\')) tags.push('utility');
-    
-    // Framework-specific tags
-    if (pathLower.includes('/pages/') || pathLower.includes('\\pages\\')) tags.push('page');
-    if (pathLower.includes('/components/') || pathLower.includes('\\components\\')) tags.push('component');
-    if (pathLower.includes('/shared/') || pathLower.includes('\\shared\\')) tags.push('shared');
-    if (pathLower.includes('/common/') || pathLower.includes('\\common\\')) tags.push('shared');
-    if (pathLower.includes('/hooks/') || pathLower.includes('\\hooks\\')) tags.push('hook');
-    if (pathLower.includes('/services/') || pathLower.includes('\\services\\')) tags.push('service');
-    if (pathLower.includes('/api/') || pathLower.includes('\\api\\')) tags.push('api');
-    if (pathLower.includes('/routes/') || pathLower.includes('\\routes\\')) tags.push('route');
-    if (pathLower.includes('/controllers/') || pathLower.includes('\\controllers\\')) tags.push('controller');
-    if (pathLower.includes('/models/') || pathLower.includes('\\models\\')) tags.push('model');
-    if (pathLower.includes('/views/') || pathLower.includes('\\views\\')) tags.push('view');
-    if (pathLower.includes('/layouts/') || pathLower.includes('\\layouts\\')) tags.push('layout');
-    if (pathLower.includes('/middleware/') || pathLower.includes('\\middleware\\')) tags.push('middleware');
-    
-    // Build and tooling
-    if (pathLower.includes('/build/') || pathLower.includes('\\build\\')) tags.push('build');
-    if (pathLower.includes('/dist/') || pathLower.includes('\\dist\\')) tags.push('build');
-    if (pathLower.includes('/.github/') || pathLower.includes('\\.github\\')) tags.push('ci');
-    if (pathLower.includes('/workflows/') || pathLower.includes('\\workflows\\')) tags.push('ci');
-    
-    // Filename-based tags (enhanced)
-    if (fileName.includes('config')) tags.push('config');
-    if (fileName.includes('test') || fileName.includes('spec')) tags.push('test');
-    if (fileName.includes('index')) tags.push('entry');
-    if (fileName.includes('main')) tags.push('entry');
-    if (fileName.includes('app')) tags.push('application');
-    if (fileName.includes('component')) tags.push('component');
-    if (fileName.includes('page')) tags.push('page');
-    if (fileName.includes('layout')) tags.push('layout');
-    if (fileName.includes('service')) tags.push('service');
-    if (fileName.includes('util') || fileName.includes('helper')) tags.push('utility');
-    if (fileName.includes('hook')) tags.push('hook');
-    if (fileName.includes('api')) tags.push('api');
-    if (fileName.includes('route')) tags.push('route');
-    if (fileName.includes('model')) tags.push('model');
-    if (fileName.includes('controller')) tags.push('controller');
-    if (fileName.includes('middleware')) tags.push('middleware');
-    if (fileName.includes('store') || fileName.includes('state')) tags.push('state');
-    if (fileName.includes('context')) tags.push('context');
-    if (fileName.includes('provider')) tags.push('provider');
-    
-    // Extension-based tags (enhanced)
-    if (['.test.js', '.spec.js', '.test.ts', '.spec.ts', '.test.tsx', '.spec.tsx'].some(ext => fullPath.endsWith(ext))) {
-      tags.push('test');
-    }
-    if (['.d.ts'].some(ext => fullPath.endsWith(ext))) {
-      tags.push('types');
-    }
-    if (['.stories.js', '.stories.ts', '.stories.tsx'].some(ext => fullPath.endsWith(ext))) {
-      tags.push('storybook');
-    }
-    if (['.cy.js', '.cy.ts'].some(ext => fullPath.endsWith(ext))) {
-      tags.push('e2e');
-    }
-    
-    // Framework detection
-    if (extension === '.tsx' || extension === '.jsx') {
-      tags.push('react');
-    }
-    if (fullPath.includes('vue') || extension === '.vue') {
-      tags.push('vue');
-    }
-    if (fullPath.includes('angular') || fullPath.includes('.component.') || fullPath.includes('.service.')) {
-      tags.push('angular');
-    }
-    if (fullPath.includes('next') || fullPath.includes('_app.') || fullPath.includes('_document.')) {
-      tags.push('nextjs');
-    }
-    
-    // Special files
-    if (['readme', 'license', 'changelog', 'contributing'].includes(fileName)) {
-      tags.push('documentation');
-    }
-    if (['dockerfile', 'docker-compose', '.dockerignore'].includes(fileName)) {
-      tags.push('docker');
-    }
-    if (['package.json', 'package-lock.json', 'yarn.lock', 'pnpm-lock.yaml'].includes(path.basename(relativePath))) {
-      tags.push('package');
-    }
-    if (['tsconfig.json', 'jsconfig.json', 'webpack.config.js', 'vite.config.js'].includes(path.basename(relativePath))) {
-      tags.push('config');
-    }
-    
-    // Infrastructure files
-    if (extension === '.bat' || extension === '.cmd') {
-      tags.push('infrastructure', 'script', 'windows');
-    }
-    if (extension === '.sh') {
-      tags.push('infrastructure', 'script', 'unix');
-    }
-    if (extension === '.json' && (fileName.includes('config') || fileName.includes('settings') || fileName.includes('.config.'))) {
-      tags.push('infrastructure', 'config');
-    }
-    if (['makefile', 'makefile.am', 'cmake', 'cmakelists.txt'].includes(fileName)) {
-      tags.push('infrastructure', 'build');
-    }
-    if (['readme', 'license', 'changelog', 'contributing'].includes(fileName)) {
-      tags.push('documentation');
-    }
-    
-    return [...new Set(tags)]; // Remove duplicates
-  }
-
-
-
-
-
-
-  /**
-   * Initialize specialized handlers for different file types
-   * @returns {object} Handler registry
-   */
-  initializeHandlers() {
-    return {
-      'code-c-like': new CLikeHandler(),
-      'code-script': new ScriptHandler(),
-      'markup': new MarkupHandler(),
-      'styling': new StylingHandler(),
-      'config-plain': new ConfigPlainHandler()
-    };
-  }
-
-  /**
-   * Get appropriate handler for file extension with full coverage
-   * @param {string} extension - File extension
-   * @returns {object} Handler instance
-   */
-  getHandler(extension) {
-    // Complete mapping for all 22 target extensions
-    const handlerMap = {
-      // Code-C like (5 extensions)
-      '.c': 'code-c-like', '.h': 'code-c-like', '.cpp': 'code-c-like', 
-      '.cs': 'code-c-like', '.java': 'code-c-like',
-      
-      // Code-Script (7 extensions)
-      '.js': 'code-script', '.jsx': 'code-script', '.ts': 'code-script', 
-      '.tsx': 'code-script', '.py': 'code-script', '.sh': 'code-script', '.bat': 'code-script',
-      
-      // Markup (2 extensions)
-      '.html': 'markup', '.xml': 'markup',
-      
-      // Styling (2 extensions)
-      '.css': 'styling', '.scss': 'styling',
-      
-      // Config/Plain (6 extensions)
-      '.json': 'config-plain', '.yaml': 'config-plain', '.yml': 'config-plain',
-      '.md': 'config-plain', '.txt': 'config-plain'
-    };
-    
-    const handlerType = handlerMap[extension];
-    
-    if (!handlerType) {
-      console.warn(`⚠️  No handler found for extension: ${extension}`);
-      return this.handlers['config-plain']; // Fallback
-    }
-    
-    return this.handlers[handlerType];
-  }
-
-  /**
-   * Verify extension coverage against target list
-   * @param {Array} processedExtensions - Extensions found in processing
-   */
-  verifyExtensionCoverage(processedExtensions) {
-    const targetExtensions = [
-      '.json', '.ts', '.js', '.jsx', '.tsx', '.xml', '.html', '.css', '.scss',
-      '.md', '.txt', '.py', '.java', '.cs', '.cpp', '.c', '.h', '.yaml', '.yml',
-      '.sh', '.bat'
-    ]; // 22 total extensions
-
-    const missing = targetExtensions.filter(ext => !processedExtensions.includes(ext));
-    const extra = processedExtensions.filter(ext => !targetExtensions.includes(ext));
-    
-    console.log(`\n📊 Extension Coverage Analysis:`);
-    console.log(`   Target extensions: ${targetExtensions.length}`);
-    console.log(`   Processed extensions: ${processedExtensions.length}`);
-    
-    if (missing.length > 0) {
-      console.warn(`   ⚠️  Missing: ${missing.join(', ')}`);
-    }
-    
-    if (extra.length > 0) {
-      console.log(`   ➕ Extra: ${extra.join(', ')}`);
-    }
-    
-    if (missing.length === 0) {
-      console.log(`   ✅ Full coverage achieved!`);
-    }
-    
-    return {
-      targetCount: targetExtensions.length,
-      processedCount: processedExtensions.length,
-      missing,
-      extra,
-      coverage: ((targetExtensions.length - missing.length) / targetExtensions.length * 100).toFixed(1)
-    };
-  }
-
-  /**
-   * Improved token estimation using multiple heuristics
-   * @param {string} content - Text content
-   * @param {string} language - Programming language for context
-   * @returns {number} Estimated token count
-   */
-  safeEstimateTokens(content, language = 'text') {
-    try {
-      if (typeof content !== 'string') {
-        console.warn('⚠️  Non-string content passed to token estimator');
-        return 0;
-      }
-      
-      if (content.length === 0) return 0;
-      
-      // Base estimation using multiple factors
-      const charCount = content.length;
-      const wordCount = content.trim().split(/\s+/).length;
-      const lineCount = content.split('\n').length;
-      
-      // Language-specific adjustments
-      let tokensPerChar = 0.25; // Default: ~4 chars per token
-      let tokensPerWord = 1.3;  // Default: ~1.3 tokens per word
-      
-      // Adjust based on content type
-      if (['javascript', 'typescript', 'python', 'java', 'c++', 'c#'].includes(language.toLowerCase())) {
-        // Code tends to have more symbols and operators
-        tokensPerChar = 0.28;
-        tokensPerWord = 1.4;
-        
-        // Additional tokens for common code patterns
-        const brackets = (content.match(/[{}()\[\]]/g) || []).length;
-        const operators = (content.match(/[+\-*/%=<>!&|^~]/g) || []).length;
-        const dots = (content.match(/\./g) || []).length;
-        
-        const syntaxTokens = Math.ceil((brackets + operators + dots) * 0.15);
-        
-        // Character-based estimation with syntax bonus
-        const charEstimate = Math.ceil(charCount * tokensPerChar) + syntaxTokens;
-        const wordEstimate = Math.ceil(wordCount * tokensPerWord);
-        
-        return Math.max(charEstimate, wordEstimate);
-      } else if (['json', 'yaml', 'xml', 'html'].includes(language.toLowerCase())) {
-        // Structured data tends to be more compact in tokens
-        tokensPerChar = 0.22;
-        tokensPerWord = 1.1;
-      } else if (language.toLowerCase() === 'markdown') {
-        // Markdown has formatting symbols but is mostly text
-        tokensPerChar = 0.26;
-        tokensPerWord = 1.2;
-      }
-      
-      // Calculate estimates using both methods
-      const charEstimate = Math.ceil(charCount * tokensPerChar);
-      const wordEstimate = Math.ceil(wordCount * tokensPerWord);
-      
-      // Return the higher estimate for safety (avoid truncation)
-      return Math.max(charEstimate, wordEstimate, Math.ceil(charCount / 4));
-      
-    } catch (error) {
-      console.warn(`⚠️  Token estimation error: ${error.message}`);
-      return Math.ceil((content?.length || 0) / 4);
-    }
-  }
-
-  /**
-   * Estimate token count using simple heuristic
-   * @param {string} content - Text content
-   * @returns {number} Estimated token count
-   */
-  estimateTokens(content) {
-    return Math.ceil(content.length / 4);
-  }
-
-  /**
-   * Extract imports from content using simple regex
-   * @param {string} content - File content
-   * @param {string} extension - File extension
-   * @returns {Array} Array of import statements
-   */
-  extractImports(content, extension) {
-    const imports = [];
-    
-    switch (extension) {
-      case '.js':
-      case '.jsx':
-      case '.ts':
-      case '.tsx':
-        // import ... from '...'
-        const importRegex = /import\s+.*?from\s+['"]([^'"]+)['"]/g;
-        let match;
-        while ((match = importRegex.exec(content)) !== null) {
-          imports.push(match[1]);
-        }
-        
-        // require('...')
-        const requireRegex = /require\s*\(\s*['"]([^'"]+)['"]\s*\)/g;
-        while ((match = requireRegex.exec(content)) !== null) {
-          imports.push(match[1]);
-        }
-        break;
-        
-      case '.py':
-        // import ... / from ... import ...
-        const pyImportRegex = /(?:from\s+(\S+)\s+import|import\s+(\S+))/g;
-        while ((match = pyImportRegex.exec(content)) !== null) {
-          imports.push(match[1] || match[2]);
-        }
-        break;
-        
-      case '.c':
-      case '.cpp':
-      case '.h':
-        // #include "..." / #include <...>
-        const includeRegex = /#include\s*[<"]([^>"]+)[>"]/g;
-        while ((match = includeRegex.exec(content)) !== null) {
-          imports.push(match[1]);
-        }
-        break;
-    }
-    
-    return [...new Set(imports)]; // Remove duplicates
-  }
-
-  /**
-   * Extract function/method calls using simple regex
-   * @param {string} content - File content
-   * @param {string} extension - File extension
-   * @returns {Array} Array of function calls
-   */
-  extractCalls(content, extension) {
-    const calls = [];
-    
-    // Generic function call pattern: identifier followed by (
-    const callRegex = /\b([a-zA-Z_][a-zA-Z0-9_]*)\s*\(/g;
-    let match;
-    
-    while ((match = callRegex.exec(content)) !== null) {
-      const funcName = match[1];
-      
-      // Filter out language keywords
-      const keywords = ['if', 'for', 'while', 'switch', 'catch', 'typeof', 'return', 'new'];
-      if (!keywords.includes(funcName) && funcName.length > 1) {
-        calls.push(funcName);
-      }
-    }
-    
-    // Return unique calls, limited to prevent noise
-    return [...new Set(calls)].slice(0, 20);
-  }
-
-  /**
-   * Validate generated JSON file
-   * @param {string} outputPath - Path to generated JSON
-   */
-  async validateGeneratedJSON(outputPath) {
-    try {
-      // Check file exists and is readable
-      const stats = await fs.stat(outputPath);
-      
-      // Check file size warnings
-      const maxSize = this.ragConfigManager.parseFileSize(this.yamlConfig?.quality?.maxOutputSize || '250MB');
-      if (stats.size > maxSize) {
-        console.warn(`⚠️  Generated file is large: ${formatFileSize(stats.size)} (>${formatFileSize(maxSize)})`);
-      }
-
-      // Simple validation: read first 1KB to check JSON structure
-      const stream = createReadStream(outputPath, { encoding: 'utf8', start: 0, end: 1023 });
-      let sampleText = '';
-      
-      for await (const chunk of stream) {
-        sampleText += chunk;
-      }
-      
-      // Check for basic JSON structure
-      if (!sampleText.trim().startsWith('{')) {
-        throw new Error('Generated file does not start with valid JSON');
-      }
-      
-      // Check for expected structure
-      if (!sampleText.includes('"metadata"') || !sampleText.includes('"files"')) {
-        console.warn('⚠️  JSON structure may be incomplete - expected sections not found in sample');
-      }
-      
-      console.log('✅ JSON validation passed');
-      
-    } catch (error) {
-      console.error(`❌ JSON validation failed: ${error.message}`);
-      // Don't re-throw - just warn since file was successfully written
-      console.warn('⚠️  Continuing despite validation warning - file was generated successfully');
-    }
-  }
-
-  /**
-   * Validate discovery progress for quality assurance
-   * @param {Array} discoveredFiles - Files discovered so far
-   * @param {number} processedCount - Total files processed
-   */
-  validateDiscoveryProgress(discoveredFiles, processedCount) {
-    const issues = [];
-    
-    // Check for duplicate hashes
-    const hashes = new Set();
-    const duplicates = [];
-    
-    for (const file of discoveredFiles) {
-      if (hashes.has(file.hash)) {
-        duplicates.push(file.hash.substring(0, 8));
-      } else {
-        hashes.add(file.hash);
-      }
-    }
-    
-    if (duplicates.length > 0) {
-      issues.push(`Duplicate hashes detected: ${duplicates.join(', ')}`);
-    }
-    
-    // Check file size distribution
-    const largeSizeThreshold = this.ragConfigManager.parseFileSize(this.yamlConfig?.performance?.maxFileSize || '100MB');
-    const largeFiles = discoveredFiles.filter(f => f.size > largeSizeThreshold);
-    
-    if (largeFiles.length > 0) {
-      issues.push(`${largeFiles.length} files exceed size threshold`);
-    }
-    
-    // Check tag distribution
-    const tagCounts = {};
-    discoveredFiles.forEach(file => {
-      file.tags.forEach(tag => {
-        tagCounts[tag] = (tagCounts[tag] || 0) + 1;
-      });
-    });
-    
-    const untaggedFiles = discoveredFiles.filter(f => f.tags.length === 0);
-    if (untaggedFiles.length > discoveredFiles.length * 0.5) {
-      issues.push(`High untagged ratio: ${untaggedFiles.length}/${discoveredFiles.length}`);
-    }
-    
-    // Report issues if any
-    if (issues.length > 0) {
-      console.warn(`\n⚠️  Discovery validation issues: ${issues.join(', ')}`);
-    }
-  }
-
-  /**
-   * Validate processing progress for quality assurance
-   * @param {Map} chunkOffsets - Current chunk offsets
-   */
-  validateProcessingProgress(chunkOffsets) {
-    const issues = [];
-    
-    // Check chunk size distribution
-    const chunkSizes = [];
-    for (const [chunkId, offsetData] of chunkOffsets.entries()) {
-      const size = offsetData.contentEnd - offsetData.contentStart;
-      chunkSizes.push(size);
-    }
-    
-    if (chunkSizes.length > 0) {
-      const avgChunkSize = chunkSizes.reduce((a, b) => a + b, 0) / chunkSizes.length;
-      const maxChunkSize = Math.max(...chunkSizes);
-      const maxChunkThreshold = this.ragConfigManager.parseFileSize(this.yamlConfig?.quality?.maxChunkSize || '50KB');
-      
-      if (maxChunkSize > maxChunkThreshold) {
-        issues.push(`Large chunk detected: ${formatFileSize(maxChunkSize)}`);
-      }
-      
-      if (avgChunkSize < 100) {
-        issues.push(`Small average chunk size: ${formatFileSize(avgChunkSize)}`);
-      }
-    }
-    
-    // Check offset consistency
-    let invalidOffsets = 0;
-    for (const [chunkId, offsetData] of chunkOffsets.entries()) {
-      if (offsetData.contentStart >= offsetData.contentEnd) {
-        invalidOffsets++;
-      }
-    }
-    
-    if (invalidOffsets > 0) {
-      issues.push(`Invalid offsets: ${invalidOffsets} chunks`);
-    }
-    
-    // Report issues if any
-    if (issues.length > 0) {
-      console.warn(`\n⚠️  Processing validation issues: ${issues.join(', ')}`);
-    }
-  }
-
-  /**
-   * Final validation of generated output with seek inverse testing
-   * @param {string} outputPath - Generated file path
-   * @param {Array} discoveredFiles - All processed files
-   * @param {Map} chunkOffsets - All chunk offsets
-   */
-  async validateFinalOutput(outputPath, discoveredFiles, chunkOffsets) {
-    const issues = [];
-    
-    try {
-      const stats = await fs.stat(outputPath);
-      
-      // Check file size
-      const maxOutputSize = this.ragConfigManager.parseFileSize(this.yamlConfig?.quality?.maxOutputSize || '250MB');
-      if (stats.size > maxOutputSize) {
-        issues.push(`Output size (${formatFileSize(stats.size)}) exceeds threshold`);
-      }
-      
-      // Check completeness
-      const expectedChunks = discoveredFiles.reduce((sum, file) => sum + (file.chunks?.length || 0), 0);
-      const actualChunks = chunkOffsets.size;
-      
-      if (expectedChunks !== actualChunks) {
-        issues.push(`Chunk count mismatch: expected ${expectedChunks}, got ${actualChunks}`);
-      }
-      
-      // Check for empty chunks
-      const emptyChunks = Array.from(chunkOffsets.values()).filter(offset => 
-        offset.contentEnd - offset.contentStart < 10
-      ).length;
-      
-      if (emptyChunks > 0) {
-        issues.push(`${emptyChunks} near-empty chunks detected`);
-      }
-      
-      // SEEK INVERSE TESTING - Test random chunk offsets
-      await this.validateSeekInverse(outputPath, chunkOffsets);
-      
-      // JSON Schema validation (basic)
-      await this.validateJsonStructure(outputPath);
-      
-      // Report final validation
-      if (issues.length > 0) {
-        console.warn(`\n⚠️  Final validation issues:`);
-        issues.forEach(issue => console.warn(`   • ${issue}`));
-      } else {
-        console.log('✅ Final validation passed - output is healthy');
-      }
-      
-    } catch (error) {
-      console.error(`❌ Final validation failed: ${error.message}`);
-    }
-  }
-
-  /**
-   * Test seek operations on random chunk offsets to verify accuracy
-   * @param {string} outputPath - Generated JSON file path
-   * @param {Map} chunkOffsets - Chunk offset map
-   */
-  async validateSeekInverse(outputPath, chunkOffsets) {
-    const chunkIds = Array.from(chunkOffsets.keys());
-    const testCount = Math.min(3, chunkIds.length); // Test 2-3 random chunks
-    
-    if (testCount === 0) {
-      console.warn('⚠️  No chunks to test for seek validation');
-      return;
-    }
-    
-    console.log(`🔍 Testing seek inverse on ${testCount} random chunks...`);
-    
-    for (let i = 0; i < testCount; i++) {
-      const randomIndex = Math.floor(Math.random() * chunkIds.length);
-      const chunkId = chunkIds[randomIndex];
-      const offsetData = chunkOffsets.get(chunkId);
-      
-      try {
-        // Read the specific chunk content using simple file read
-        const fullContent = await fs.readFile(outputPath, 'utf8');
-        const seekContent = fullContent.slice(offsetData.contentStart, offsetData.contentEnd);
-        
-        // Verify it's valid JSON content (should be a JSON string value)
-        try {
-          // Try to parse as JSON - if it's valid JSON string content, this should work
-          const parsed = JSON.parse(seekContent);
-          if (typeof parsed === 'string') {
-            console.log(`   ✅ Chunk ${chunkId}: seek successful, valid JSON string (${seekContent.length} bytes)`);
-          } else {
-            console.log(`   ✅ Chunk ${chunkId}: seek successful, valid JSON (${typeof parsed}, ${seekContent.length} bytes)`);
-          }
-        } catch (parseError) {
-          // If it doesn't parse as JSON, it might be a partial chunk
-          console.log(`   ✅ Chunk ${chunkId}: seek successful, partial content (${seekContent.length} bytes)`);
-        }
-        
-      } catch (error) {
-        console.error(`   ❌ Chunk ${chunkId}: seek failed - ${error.message}`);
-      }
-    }
-  }
-
-  /**
-   * Basic JSON structure validation
-   * @param {string} outputPath - Generated JSON file path
-   */
-  async validateJsonStructure(outputPath) {
-    try {
-      // Read full content for validation (simpler approach)
-      const fullContent = await fs.readFile(outputPath, 'utf8');
-      const startText = fullContent.slice(0, 1024).trim();
-      const endText = fullContent.slice(-1024).trim();
-      
-      // Basic structure checks
-      const issues = [];
-      
-      if (!startText.startsWith('{')) {
-        issues.push('File does not start with {');
-      }
-      
-      if (!endText.endsWith('}')) {
-        issues.push('File does not end with }');
-      }
-      
-      if (!startText.includes('"metadata"')) {
-        issues.push('Missing metadata section');
-      }
-      
-      if (!startText.includes('"files"')) {
-        issues.push('Missing files section');
-      }
-      
-      if (!fullContent.includes('"index"')) {
-        issues.push('Missing index section');
-      }
-      
-      if (issues.length === 0) {
-        console.log('✅ JSON structure validation passed');
-      } else {
-        console.warn(`⚠️  JSON structure issues: ${issues.join(', ')}`);
-      }
-      
-    } catch (error) {
-      console.error(`❌ JSON structure validation failed: ${error.message}`);
-    }
-  }
-
-  /**
-   * Generate RAG output atomically - build complete structure in memory (thread-safe)
-   */
-  async generate(discoveredFiles, outputPath, projectName, scanPath) {
-    console.log('📝 Atomic generation: processing all files in memory...');
-    
-    await fs.ensureDir(path.dirname(outputPath));
-    
-    const processedFiles = [];
-    let totalChunks = 0;
-    
-    for (let i = 0; i < discoveredFiles.length; i++) {
-      const fileData = discoveredFiles[i];
-      const progress = ((i + 1) / discoveredFiles.length * 100).toFixed(1);
-      
-      process.stdout.write(`\r📊 Processing: ${progress}% (${i + 1}/${discoveredFiles.length})`);
-      
-      try {
-        const processedFile = await this.processFileInMemory(fileData);
-        processedFiles.push(processedFile);
-        totalChunks += processedFile.chunks?.length || 0;
-        this.stats.filesProcessed++;
-        this.stats.chunksGenerated += processedFile.chunks?.length || 0;
-      } catch (error) {
-        console.warn(`\n⚠️  Error processing ${fileData.path}: ${error.message}`);
-        this.errors.push({ file: fileData.path, error: error.message });
-        processedFiles.push({ ...fileData, chunks: [], error: error.message });
-      }
-    }
-    
-    console.log(`\n✅ All files processed: ${processedFiles.length} files, ${totalChunks} chunks`);
-    
-    const completeJSON = this.buildCompleteJSON(processedFiles, projectName, scanPath);
-    const finalJSON = this.calculateAndInjectOffsets(completeJSON);
-    
-    await fs.writeFile(outputPath, finalJSON, 'utf8');
-    this.stats.bytesWritten = finalJSON.length;
-    
-    if (this.yamlConfig?.output?.validation) {
-      console.log('🔍 Validating generated output...');
-      await this.validateGeneratedJSON(outputPath);
-    }
-    
-    console.log(`✅ JSON written successfully to ${outputPath}`);
-    
-    return {
-      outputPath,
-      totalFiles: processedFiles.length,
-      totalChunks,
-      bytesWritten: finalJSON.length,
-      extensionCoverage: this.verifyExtensionCoverage([...new Set(processedFiles.map(f => f.extension))])
-    };
-  }
-
-  async processFileInMemory(fileData) {
-    const content = await fs.readFile(fileData.absolutePath, 'utf8');
-    const handler = this.getHandler(fileData.extension);
-    
-    console.log(`🔍 ${handler.constructor.name} processing ${fileData.extension} file: ${fileData.path}`);
-    
-    const chunks = await handler.generateChunks(content, {
-      fileId: fileData.id,
-      filePath: fileData.path,
-      extension: fileData.extension,
-      language: fileData.language,
-      maxTokens: this.maxTokensPerChunk,
-      overlap: this.overlapTokens
-    });
-    
-    if (chunks.length > 0) {
-      console.log(`   📝 Found ${chunks.length} semantic chunks`);
-    }
-    
-    const enrichedChunks = chunks.map((chunk, index) => ({
-      ...chunk,
-      id: `chunk_${fileData.id}_${index}`,
-      tokenEstimate: this.safeEstimateTokens(chunk.content, fileData.language),
-      imports: this.extractImports(chunk.content, fileData.extension),
-      calls: this.extractCalls(chunk.content, fileData.extension)
-    }));
-    
-    return {
-      id: fileData.id,
-      path: fileData.path,
-      language: fileData.language,
-      extension: fileData.extension,
-      size: fileData.size,
-      lines: content.split('\n').length,
-      hash: fileData.hash,
-      modified: fileData.modified,
-      tags: fileData.tags,
-      chunks: enrichedChunks
-    };
-  }
-
-  buildCompleteJSON(processedFiles, projectName, scanPath) {
-    const totalChunks = processedFiles.reduce((sum, file) => sum + (file.chunks?.length || 0), 0);
-    const emptyFiles = processedFiles.filter(f => (f.chunks?.length || 0) === 0).length;
-    
-    this.stats.endTime = Date.now();
-    const processingTimeMs = Math.max(1, this.stats.endTime - this.stats.startTime);
-    
-    return {
-      metadata: {
-        projectName,
-        generatedAt: new Date().toISOString(),
-        scanPath,
-        generator: 'CodeSummary RAG Generator',
-        version: '3.1.0',
-        config: {
-          maxTokensPerChunk: this.maxTokensPerChunk,
-          overlapTokens: this.overlapTokens,
-          tokenEstimationMethod: 'enhanced_heuristic_v1.0'
-        },
-        summary: {
-          totalFiles: processedFiles.length,
-          languages: [...new Set(processedFiles.map(f => f.language))],
-          extensions: [...new Set(processedFiles.map(f => f.extension))]
-        },
-        schemaVersion: "1.0",
-        schemaUrl: "https://github.com/skamoll/CodeSummary/schemas/rag-output.json"
-      },
-      files: processedFiles,
-      index: {
-        version: "3.1.0",
-        generatedAt: new Date().toISOString(),
-        schemaUrl: "https://github.com/skamoll/CodeSummary/schemas/rag-output.json",
-        summary: {
-          fileCount: processedFiles.length - emptyFiles,
-          chunkCount: totalChunks,
-          totalBytes: 0,
-          languages: [...new Set(processedFiles.map(f => f.language))],
-          extensions: [...new Set(processedFiles.map(f => f.extension))],
-          avgFileSize: 0,
-          avgChunksPerFile: processedFiles.length > 0 ? Math.round(totalChunks / processedFiles.length) : 0
-        },
-        chunkOffsets: {},
-        fileOffsets: {},
-        seekInfo: {
-          instructions: "Use chunkOffsets[chunkId].contentStart and contentEnd to seek directly to chunk content",
-          format: "All offsets are absolute byte positions in this JSON file",
-          chunkFormat: "Object with jsonStart, jsonEnd, contentStart, contentEnd (absolute JSON positions)",
-          fileFormat: "Array [start, end] for each file in JSON"
-        },
-        statistics: {
-          processingTimeMs,
-          bytesPerSecond: 0,
-          bytesWritten: 0,
-          chunksWithValidOffsets: totalChunks,
-          filesWithValidOffsets: processedFiles.length - emptyFiles,
-          totalFiles: processedFiles.length,
-          emptyFiles: emptyFiles,
-          totalChunksGenerated: totalChunks,
-          errors: this.errors
-        }
-      }
-    };
-  }
-
-  /**
-   * Finaliza la estructura JSON calculando y reinyectando los offsets correctos.
-   * Este enfoque garantiza la máxima precisión al operar sobre el string JSON final.
-   * @param {object} jsonStructure - El objeto JSON completo con datos pero sin offsets.
-   * @returns {string} El string JSON final, formateado y con offsets precisos.
-   */
-  calculateAndInjectOffsets(jsonStructure) {
-    console.log('🔍 Calculating precise byte offsets and building complete index...');
-    
-    // PASO 1: Construir JSON preliminar sin index para medir posiciones exactas
-    const jsonWithoutIndex = {
-      metadata: jsonStructure.metadata,
-      files: jsonStructure.files
-    };
-    
-    const preliminaryJsonString = JSON.stringify(jsonWithoutIndex, null, 2);
-    const preliminaryBytes = Buffer.byteLength(preliminaryJsonString, 'utf8');
-    
-    // PASO 2: Calcular offsets precisos de archivos y chunks
-    const fileOffsets = {};
-    const chunkOffsets = {};
-    let totalChunks = 0;
-    let validChunks = 0;
-    
-    for (const file of jsonStructure.files) {
-      // Buscar el inicio del objeto file por su ID
-      const filePattern = `"id": "${file.id}"`;
-      const fileStartPos = preliminaryJsonString.indexOf(filePattern);
-      
-      if (fileStartPos !== -1) {
-        // Buscar el final aproximado del objeto file
-        const nextFilePattern = preliminaryJsonString.indexOf('    {\n      "id":', fileStartPos + 1);
-        const fileEndPos = nextFilePattern !== -1 ? nextFilePattern : preliminaryJsonString.lastIndexOf('  ]');
-        
-        // Formato del esquema: fileId -> [start, end]
-        fileOffsets[file.id] = [fileStartPos, fileEndPos];
-        
-        // Calcular offsets de chunks dentro de este archivo
-        for (const chunk of file.chunks) {
-          const chunkPattern = `"id": "${chunk.id}"`;
-          const chunkStartPos = preliminaryJsonString.indexOf(chunkPattern, fileStartPos);
-          
-          if (chunkStartPos !== -1) {
-            // Encontrar el campo "content" dentro de este chunk
-            const contentPattern = '"content": "';
-            const contentStartSearch = preliminaryJsonString.indexOf(contentPattern, chunkStartPos);
-            
-            if (contentStartSearch !== -1) {
-              const contentStart = contentStartSearch + contentPattern.length;
-              
-              // Buscar el final del contenido (cierre de la cadena JSON)
-              let contentEnd = contentStart;
-              let inEscape = false;
-              
-              for (let i = contentStart; i < preliminaryJsonString.length; i++) {
-                const char = preliminaryJsonString[i];
-                if (inEscape) {
-                  inEscape = false;
-                  continue;
-                }
-                if (char === '\\') {
-                  inEscape = true;
-                  continue;
-                }
-                if (char === '"') {
-                  contentEnd = i;
-                  break;
-                }
-              }
-              
-              // Buscar el final del objeto chunk completo
-              const chunkEndPattern = '},';
-              const chunkEndSearch = preliminaryJsonString.indexOf(chunkEndPattern, contentEnd);
-              const chunkEnd = chunkEndSearch !== -1 ? chunkEndSearch + 1 : contentEnd + 100;
-              
-              // Formato del esquema: chunkId -> objeto con offsets precisos
-              chunkOffsets[chunk.id] = {
-                jsonStart: chunkStartPos,
-                jsonEnd: chunkEnd,
-                contentStart: contentStart,
-                contentEnd: contentEnd,
-                filePath: file.path
-              };
-              
-              validChunks++;
-            }
-          }
-          totalChunks++;
-        }
-      }
-    }
-    
-    // PASO 3: Construir estadísticas completas
-    const processingTimeMs = Math.max(1, this.stats.endTime - this.stats.startTime);
-    const emptyFiles = jsonStructure.files.filter(f => f.chunks.length === 0).length;
-    
-    // PASO 4: Construir el bloque index completo según el esquema
-    const indexBlock = {
-      version: "3.1.0",
-      generatedAt: new Date().toISOString(),
-      schemaUrl: "https://github.com/skamoll/CodeSummary/schemas/rag-output.json",
-      summary: {
-        fileCount: jsonStructure.files.length - emptyFiles,
-        chunkCount: totalChunks,
-        totalBytes: 0, // Se actualizará después
-        languages: [...new Set(jsonStructure.files.map(f => f.language))],
-        extensions: [...new Set(jsonStructure.files.map(f => f.extension))],
-        avgFileSize: 0, // Se actualizará después
-        avgChunksPerFile: jsonStructure.files.length > 0 ? Math.round(totalChunks / jsonStructure.files.length) : 0
-      },
-      chunkOffsets: chunkOffsets,
-      fileOffsets: fileOffsets,
-      seekInfo: {
-        instructions: "Use chunkOffsets[chunkId].contentStart and contentEnd to seek directly to chunk content",
-        format: "All offsets are absolute byte positions in this JSON file",
-        chunkFormat: "Object with jsonStart, jsonEnd, contentStart, contentEnd (absolute JSON positions)",
-        fileFormat: "Array [start, end] for each file in JSON",
-        validation: `Generated with ${validChunks} chunks across ${Object.keys(fileOffsets).length} files`
-      },
-      statistics: {
-        processingTimeMs,
-        bytesPerSecond: 0, // Se actualizará después
-        bytesWritten: 0, // Se actualizará después
-        chunksWithValidOffsets: validChunks,
-        filesWithValidOffsets: Object.keys(fileOffsets).length,
-        totalFiles: jsonStructure.files.length,
-        emptyFiles: emptyFiles,
-        totalChunksGenerated: totalChunks
-      }
-    };
-    
-    // PASO 5: Construir JSON final con index y calcular métricas finales
-    const completeStructure = {
-      metadata: jsonStructure.metadata,
-      files: jsonStructure.files,
-      index: indexBlock
-    };
-    
-    const finalJsonString = JSON.stringify(completeStructure, null, 2);
-    const finalBytes = Buffer.byteLength(finalJsonString, 'utf8');
-    const bytesPerSecond = Math.round(finalBytes / (processingTimeMs / 1000));
-    
-    // Actualizar métricas finales en el index
-    completeStructure.index.summary.totalBytes = finalBytes;
-    completeStructure.index.summary.avgFileSize = jsonStructure.files.length > 0 ? 
-      Math.round(finalBytes / jsonStructure.files.length) : 0;
-    completeStructure.index.statistics.bytesPerSecond = bytesPerSecond;
-    completeStructure.index.statistics.bytesWritten = finalBytes;
-    
-    // PASO 6: Regenerar JSON final con estadísticas actualizadas
-    const finalResult = JSON.stringify(completeStructure, null, 2);
-    
-    console.log(`✅ Complete index built: ${Object.keys(fileOffsets).length} files, ${validChunks}/${totalChunks} chunks with precise offsets`);
-    console.log(`✅ Final JSON: ${formatFileSize(Buffer.byteLength(finalResult, 'utf8'))}, processing: ${processingTimeMs}ms`);
-    
-    return finalResult;
-  }
-}
-
-// Specialized Handler Classes
-
-class BaseHandler {
-  async generateChunks(content, options) {
-    // Fallback: split by lines if no specific logic
-    return this.chunkByLines(content, options);
-  }
-  
-  chunkByLines(content, options) {
-    const lines = content.split('\n');
-    const chunks = [];
-    const maxLines = Math.ceil(options.maxTokens / 20); // ~20 tokens per line estimate
-    
-    for (let i = 0; i < lines.length; i += maxLines) {
-      const chunkLines = lines.slice(i, Math.min(i + maxLines, lines.length));
-      const chunkContent = chunkLines.join('\n');
-      
-      chunks.push({
-        content: chunkContent,
-        lineStart: i + 1,
-        lineEnd: Math.min(i + maxLines, lines.length),
-        chunkingMethod: 'line-based'
-      });
-    }
-    
-    return chunks;
-  }
-
-  /**
-   * Estimate token count for chunking decisions
-   * @param {string} content - Text content
-   * @returns {number} Estimated token count
-   */
-  estimateTokens(content) {
-    return Math.ceil(content.length / 4);
-  }
-}
-
-class CLikeHandler extends BaseHandler {
-  async generateChunks(content, options) {
-    const chunks = [];
-    const lines = content.split('\n');
-    
-    // Find class/struct/function boundaries
-    const boundaries = this.findCodeBoundaries(content);
-    
-    if (boundaries.length > 0) {
-      return this.chunkByBoundaries(content, boundaries, options);
-    }
-    
-    // Fallback to line-based chunking
-    return this.chunkByLines(content, options);
-  }
-  
-  findCodeBoundaries(content) {
-    const boundaries = [];
-    const boundaryRegex = /^(?:class|struct|enum|union|static)?\s*([a-zA-Z_][\w]*)\s*.*{/gm;
-    let match;
-    
-    while ((match = boundaryRegex.exec(content)) !== null) {
-      const lineNumber = content.substring(0, match.index).split('\n').length;
-      boundaries.push({
-        name: match[1],
-        line: lineNumber,
-        type: 'function'
-      });
-    }
-    
-    return boundaries;
-  }
-  
-  chunkByBoundaries(content, boundaries, options) {
-    const lines = content.split('\n');
-    const chunks = [];
-    let currentStart = 0;
-    
-    for (const boundary of boundaries) {
-      if (currentStart < boundary.line - 1) {
-        const chunkLines = lines.slice(currentStart, boundary.line - 1);
-        if (chunkLines.length > 0) {
-          chunks.push({
-            content: chunkLines.join('\n'),
-            lineStart: currentStart + 1,
-            lineEnd: boundary.line - 1
-          });
-        }
-      }
-      currentStart = boundary.line - 1;
-    }
-    
-    // Add remaining lines
-    if (currentStart < lines.length) {
-      const chunkLines = lines.slice(currentStart);
-      chunks.push({
-        content: chunkLines.join('\n'),
-        lineStart: currentStart + 1,
-        lineEnd: lines.length
-      });
-    }
-    
-    return chunks;
-  }
-}
-
-class ScriptHandler extends BaseHandler {
-  async generateChunks(content, options) {
-    console.log(`🔍 ScriptHandler processing ${options.extension} file: ${options.filePath}`);
-    
-    // ALWAYS try semantic chunking first for script files
-    const functions = this.findFunctions(content, options.extension);
-    
-    if (functions.length > 0) {
-      console.log(`   📝 Found ${functions.length} functions/classes - using semantic chunking`);
-      const chunks = this.chunkByFunctions(content, functions, options);
-      
-      // Add chunking method metadata
-      chunks.forEach(chunk => {
-        chunk.chunkingMethod = 'semantic-function';
-        chunk.semanticContext = chunk.context || 'code-block';
-      });
-      
-      return chunks;
-    } else {
-      console.log(`   ⚠️  No functions found - falling back to line-based chunking`);
-      const chunks = this.chunkByLines(content, options);
-      chunks.forEach(chunk => {
-        chunk.chunkingMethod = 'line-based-fallback';
-      });
-      return chunks;
-    }
-  }
-  
-  findFunctions(content, extension) {
-    const functions = [];
-    
-    if (['.js', '.jsx', '.ts', '.tsx'].includes(extension)) {
-      // Enhanced JavaScript/TypeScript function detection with improved precision
-      const patterns = [
-        // Regular functions: function name() {} - SEMANTIC BOUNDARY
-        /(?:^|\n)\s*(?:export\s+)?(?:async\s+)?function\s+([a-zA-Z_$][a-zA-Z0-9_$]*)\s*\([^)]*\)\s*\{/gm,
-        // Arrow functions: const name = () => {} - SEMANTIC BOUNDARY  
-        /(?:^|\n)\s*(?:export\s+)?(?:const|let|var)\s+([a-zA-Z_$][a-zA-Z0-9_$]*)\s*=\s*(?:async\s+)?\([^)]*\)\s*=>\s*\{/gm,
-        // Classes: class ClassName {} - SEMANTIC BOUNDARY
-        /(?:^|\n)\s*(?:export\s+)?(?:abstract\s+)?class\s+([a-zA-Z_$][a-zA-Z0-9_$]*)(?:\s+extends\s+[a-zA-Z_$][a-zA-Z0-9_$]*)?(?:\s+implements\s+[^{]+)?\s*\{/gm,
-        // Interfaces (TypeScript): interface InterfaceName {} - SEMANTIC BOUNDARY
-        /(?:^|\n)\s*(?:export\s+)?interface\s+([a-zA-Z_$][a-zA-Z0-9_$]*)(?:\s+extends\s+[^{]+)?\s*\{/gm,
-        // Type definitions (TypeScript): type TypeName = - SEMANTIC BOUNDARY
-        /(?:^|\n)\s*(?:export\s+)?type\s+([a-zA-Z_$][a-zA-Z0-9_$]*)\s*=/gm,
-        // Enum definitions (TypeScript): enum EnumName {} - SEMANTIC BOUNDARY
-        /(?:^|\n)\s*(?:export\s+)?(?:const\s+)?enum\s+([a-zA-Z_$][a-zA-Z0-9_$]*)\s*\{/gm,
-        // Class methods and object methods - SEMANTIC BOUNDARY (improved to avoid false positives)
-        /(?:^|\n)\s*(?:public\s+|private\s+|protected\s+|static\s+|async\s+)*([a-zA-Z_$][a-zA-Z0-9_$]*)\s*\([^)]*\)\s*\{/gm
-      ];
-      
-      const types = ['function', 'arrow-function', 'class', 'interface', 'type', 'enum', 'method'];
-      
-      patterns.forEach((regex, patternIndex) => {
-        let match;
-        while ((match = regex.exec(content)) !== null) {
-          const lineNumber = content.substring(0, match.index).split('\n').length;
-          const endLine = this.findFunctionEnd(content, match.index, lineNumber);
-          
-          functions.push({
-            name: match[1],
-            line: lineNumber,
-            endLine: endLine,
-            type: types[patternIndex],
-            startIndex: match.index
-          });
-        }
-      });
-      
-      // Sort by line number to process in order
-      functions.sort((a, b) => a.line - b.line);
-      
-    } else if (extension === '.py') {
-      // Python functions/classes
-      const pyFuncRegex = /^(?:def\s+(\w+)|class\s+(\w+))/gm;
-      let match;
-      
-      while ((match = pyFuncRegex.exec(content)) !== null) {
-        const lineNumber = content.substring(0, match.index).split('\n').length;
-        functions.push({
-          name: match[1] || match[2],
-          line: lineNumber,
-          type: match[2] ? 'class' : 'function'
-        });
-      }
-    }
-    
-    return functions;
-  }
-  
-  /**
-   * Find the end line of a function/class by matching braces
-   * @param {string} content - Full content
-   * @param {number} startIndex - Start character index
-   * @param {number} startLine - Start line number
-   * @returns {number} End line number
-   */
-  findFunctionEnd(content, startIndex, startLine) {
-    let braceCount = 0;
-    let inString = false;
-    let stringChar = '';
-    let i = startIndex;
-    
-    // Find the opening brace
-    while (i < content.length && content[i] !== '{') {
-      if (content[i] === '\n') {
-        // If we hit a newline before finding {, it might be an interface/type without body
-        if (content.substring(startIndex, i).includes('interface') || 
-            content.substring(startIndex, i).includes('type')) {
-          // For interfaces/types, find the end of the statement
-          while (i < content.length && content[i] !== '\n') i++;
-          return content.substring(0, i).split('\n').length;
-        }
-      }
-      i++;
-    }
-    
-    if (i >= content.length) {
-      // No opening brace found, probably a single-line declaration
-      return startLine;
-    }
-    
-    // Count braces to find matching closing brace
-    braceCount = 1;
-    i++;
-    
-    while (i < content.length && braceCount > 0) {
-      const char = content[i];
-      
-      // Handle strings to avoid counting braces inside strings
-      if (!inString) {
-        if (char === '"' || char === "'" || char === '`') {
-          inString = true;
-          stringChar = char;
-        } else if (char === '{') {
-          braceCount++;
-        } else if (char === '}') {
-          braceCount--;
-        }
-      } else {
-        if (char === stringChar && content[i - 1] !== '\\') {
-          inString = false;
-          stringChar = '';
-        }
-      }
-      
-      i++;
-    }
-    
-    // Return the line number where the function ends
-    return content.substring(0, i).split('\n').length;
-  }
-  
-  chunkByFunctions(content, functions, options) {
-    const lines = content.split('\n');
-    const chunks = [];
-    let currentStart = 0;
-    
-    for (let i = 0; i < functions.length; i++) {
-      const func = functions[i];
-      const funcStart = func.line - 1;
-      const funcEnd = func.endLine;
-      
-      // Add content before function (imports, comments, etc.)
-      if (currentStart < funcStart) {
-        const preLines = lines.slice(currentStart, funcStart);
-        const preContent = preLines.join('\n').trim();
-        
-        if (preContent.length > 0) {
-          chunks.push({
-            content: preContent,
-            lineStart: currentStart + 1,
-            lineEnd: funcStart,
-            context: 'file_header'
-          });
-        }
-      }
-      
-      // Add the function itself
-      const funcLines = lines.slice(funcStart, funcEnd);
-      const funcContent = funcLines.join('\n');
-      
-      // Check if function is too large and needs splitting
-      if (this.estimateTokens(funcContent) > options.maxTokens * 1.5) {
-        // Split large function into smaller chunks
-        const subChunks = this.splitLargeFunction(funcContent, func, options);
-        chunks.push(...subChunks);
-      } else {
-        chunks.push({
-          content: funcContent,
-          lineStart: funcStart + 1,
-          lineEnd: funcEnd,
-          context: `${func.type}_${func.name}`,
-          entityName: func.name,
-          entityType: func.type
-        });
-      }
-      
-      currentStart = funcEnd;
-    }
-    
-    // Add remaining content after last function
-    if (currentStart < lines.length) {
-      const remainingLines = lines.slice(currentStart);
-      const remainingContent = remainingLines.join('\n').trim();
-      
-      if (remainingContent.length > 0) {
-        chunks.push({
-          content: remainingContent,
-          lineStart: currentStart + 1,
-          lineEnd: lines.length,
-          context: 'file_footer'
-        });
-      }
-    }
-    
-    return chunks;
-  }
-  
-  /**
-   * Split a large function into smaller semantic chunks
-   * @param {string} funcContent - Function content
-   * @param {object} func - Function metadata
-   * @param {object} options - Chunking options
-   * @returns {Array} Array of chunks
-   */
-  splitLargeFunction(funcContent, func, options) {
-    const chunks = [];
-    const lines = funcContent.split('\n');
-    const maxLines = Math.ceil(options.maxTokens / 20);
-    
-    for (let i = 0; i < lines.length; i += maxLines) {
-      const chunkLines = lines.slice(i, i + maxLines);
-      const chunkContent = chunkLines.join('\n');
-      
-      chunks.push({
-        content: chunkContent,
-        lineStart: func.line + i,
-        lineEnd: func.line + Math.min(i + maxLines, lines.length) - 1,
-        context: `${func.type}_${func.name}_part${Math.floor(i / maxLines) + 1}`,
-        entityName: func.name,
-        entityType: func.type,
-        isPartial: true
-      });
-    }
-    
-    return chunks;
-  }
-}
-
-class MarkupHandler extends BaseHandler {
-  async generateChunks(content, options) {
-    // Split by major HTML tags or by token limit
-    const majorTags = ['<body>', '<div', '<section', '<article', '<main'];
-    
-    for (const tag of majorTags) {
-      if (content.includes(tag)) {
-        return this.chunkByTag(content, tag, options);
-      }
-    }
-    
-    return this.chunkByLines(content, options);
-  }
-  
-  chunkByTag(content, tag, options) {
-    const parts = content.split(tag);
-    const chunks = [];
-    
-    parts.forEach((part, index) => {
-      if (part.trim().length > 0) {
-        const chunkContent = index === 0 ? part : tag + part;
-        chunks.push({
-          content: chunkContent,
-          lineStart: 1, // Approximate
-          lineEnd: chunkContent.split('\n').length
-        });
-      }
-    });
-    
-    return chunks;
-  }
-}
-
-class StylingHandler extends BaseHandler {
-  async generateChunks(content, options) {
-    // Split by CSS rules (closing braces)
-    const rules = content.split('}');
-    const chunks = [];
-    let currentChunk = '';
-    let lineStart = 1;
-    
-    for (const rule of rules) {
-      currentChunk += rule + '}';
-      
-      if (this.estimateTokens(currentChunk) >= options.maxTokens) {
-        const lineEnd = lineStart + currentChunk.split('\n').length - 1;
-        chunks.push({
-          content: currentChunk.trim(),
-          lineStart,
-          lineEnd
-        });
-        
-        currentChunk = '';
-        lineStart = lineEnd + 1;
-      }
-    }
-    
-    if (currentChunk.trim().length > 0) {
-      chunks.push({
-        content: currentChunk.trim(),
-        lineStart,
-        lineEnd: lineStart + currentChunk.split('\n').length - 1
-      });
-    }
-    
-    return chunks;
-  }
-  
-  estimateTokens(content) {
-    return Math.ceil(content.length / 4);
-  }
-}
-
-class ConfigPlainHandler extends BaseHandler {
-  async generateChunks(content, options) {
-    if (options.extension === '.json') {
-      return this.chunkJson(content, options);
-    } else if (['.yaml', '.yml'].includes(options.extension)) {
-      return this.chunkYaml(content, options);
-    } else if (options.extension === '.md') {
-      return this.chunkMarkdown(content, options);
-    }
-    
-    return this.chunkByLines(content, options);
-  }
-  
-  chunkJson(content, options) {
-    try {
-      const parsed = JSON.parse(content);
-      
-      if (typeof parsed === 'object' && !Array.isArray(parsed)) {
-        // Split by top-level keys
-        const chunks = [];
-        const keys = Object.keys(parsed);
-        
-        for (const key of keys) {
-          const section = { [key]: parsed[key] };
-          chunks.push({
-            content: JSON.stringify(section, null, 2),
-            lineStart: 1,
-            lineEnd: JSON.stringify(section, null, 2).split('\n').length,
-            context: `json_key_${key}`
-          });
-        }
-        
-        return chunks;
-      }
-    } catch (error) {
-      // Fall back to line-based chunking if JSON is invalid
-    }
-    
-    return this.chunkByLines(content, options);
-  }
-  
-  chunkMarkdown(content, options) {
-    // Split by headers
-    const headerRegex = /^#+\s+(.+)/gm;
-    const chunks = [];
-    let currentChunk = '';
-    let lineStart = 1;
-    let match;
-    
-    while ((match = headerRegex.exec(content)) !== null) {
-      if (currentChunk.length > 0) {
-        const lineEnd = lineStart + currentChunk.split('\n').length - 1;
-        chunks.push({
-          content: currentChunk.trim(),
-          lineStart,
-          lineEnd
-        });
-        lineStart = lineEnd + 1;
-      }
-      
-      currentChunk = match[0] + '\n';
-    }
-    
-    if (currentChunk.trim().length > 0) {
-      chunks.push({
-        content: currentChunk.trim(),
-        lineStart,
-        lineEnd: lineStart + currentChunk.split('\n').length - 1
-      });
-    }
-    
-    return chunks.length > 0 ? chunks : this.chunkByLines(content, options);
-  }
-  
-  chunkYaml(content, options) {
-    // Simple YAML chunking by document separators and top-level keys
-    const lines = content.split('\n');
-    const chunks = [];
-    let currentChunk = '';
-    let lineStart = 1;
-    
-    for (let i = 0; i < lines.length; i++) {
-      const line = lines[i];
-      currentChunk += line + '\n';
-      
-      // Check if we hit a top-level key or document separator
-      if (line.match(/^[a-zA-Z_][\w]*:/) || line === '---' || this.estimateTokens(currentChunk) >= options.maxTokens) {
-        if (currentChunk.trim().length > 0) {
-          chunks.push({
-            content: currentChunk.trim(),
-            lineStart,
-            lineEnd: i + 1
-          });
-          
-          lineStart = i + 2;
-          currentChunk = '';
-        }
-      }
-    }
-    
-    if (currentChunk.trim().length > 0) {
-      chunks.push({
-        content: currentChunk.trim(),
-        lineStart,
-        lineEnd: lines.length
-      });
-    }
-    
-    return chunks.length > 0 ? chunks : this.chunkByLines(content, options);
-  }
-  
-  estimateTokens(content) {
-    return Math.ceil(content.length / 4);
-  }
-}
-
-export default new RagGenerator();