codecritique 1.0.0 → 1.1.0

package/src/embeddings/file-processor.js

@@ -1,851 +0,0 @@
-/**
- * File Processor Module
- *
- * This module provides centralized file processing capabilities for embeddings.
- * It handles batch processing, directory structure generation, and progress tracking.
- *
- * Features:
- * - Batch file processing with progress tracking
- * - Directory structure generation and embedding
- * - File filtering and exclusion logic
- * - Document chunk processing
- * - Vector index creation
- * - Comprehensive error handling
- */
-
-import { createHash } from 'node:crypto';
-import fs from 'node:fs';
-import path from 'node:path';
-import chalk from 'chalk';
-import { isDocumentationFile, shouldProcessFile as utilsShouldProcessFile, batchCheckGitignore } from '../utils/file-validation.js';
-import { detectLanguageFromExtension } from '../utils/language-detection.js';
-import { debug } from '../utils/logging.js';
-import { extractMarkdownChunks } from '../utils/markdown.js';
-import { slugify } from '../utils/string-utils.js';
-import { TABLE_NAMES, LANCEDB_DIR_NAME, FASTEMBED_CACHE_DIR_NAME } from './constants.js';
-import { createFileProcessingError } from './errors.js';
-
-// ============================================================================
-// FILE PROCESSOR CLASS
-// ============================================================================
-
-export class FileProcessor {
-  constructor(options = {}) {
-    this.modelManager = options.modelManager || null;
-    this.databaseManager = options.databaseManager || null;
-    this.cacheManager = options.cacheManager || null;
-
-    // Processing state
-    this.processedFiles = new Map();
-    this.cleaningUp = false;
-    this.progressTracker = {
-      totalFiles: 0,
-      processedCount: 0,
-      skippedCount: 0,
-      failedCount: 0,
-      startTime: 0,
-      reset(total) {
-        this.totalFiles = total;
-        this.processedCount = 0;
-        this.skippedCount = 0;
-        this.failedCount = 0;
-        this.startTime = Date.now();
-      },
-      update(type) {
-        if (type === 'processed') this.processedCount++;
-        if (type === 'skipped') this.skippedCount++;
-        if (type === 'failed') this.failedCount++;
-        // Progress logging is now handled by the spinner in index.js via onProgress callback
-      },
-    };
-
-    // Table names
-    this.fileEmbeddingsTable = TABLE_NAMES.FILE_EMBEDDINGS;
-    this.documentChunkTable = TABLE_NAMES.DOCUMENT_CHUNK;
-  }
-
-  // ============================================================================
-  // PROGRESS TRACKING
-  // ============================================================================
-
-  /**
-   * Get progress tracker
-   * @returns {Object} Progress tracker object
-   */
-  getProgressTracker() {
-    return this.progressTracker;
-  }
-
-  /**
-   * Reset progress tracker
-   * @param {number} totalFiles - Total number of files to process
-   */
-  resetProgressTracker(totalFiles = 0) {
-    this.progressTracker.reset(totalFiles);
-  }
-
-  // ============================================================================
-  // DIRECTORY STRUCTURE PROCESSING
-  // ============================================================================
-
-  /**
-   * Generate directory structure string
-   * @param {Object} options - Options for generating directory structure
-   * @returns {string} Directory structure as a string
-   */
-  generateDirectoryStructure(options = {}) {
-    const { rootDir = process.cwd(), maxDepth = 5, ignorePatterns = [], showFiles = true } = options;
-    debug(`Generating directory structure: rootDir=${rootDir}, maxDepth=${maxDepth}, showFiles=${showFiles}`);
-
-    // Use path.sep for platform compatibility
-    const pathSep = path.sep;
-    // More robust ignore pattern matching (handles directory separators)
-    const shouldIgnore = (relPath) =>
-      ignorePatterns.some((pattern) => {
-        const normalizedPattern = pattern.replace(/\//g, pathSep); // Normalize pattern separators
-        const normalizedPath = relPath.replace(/\//g, pathSep);
-        if (normalizedPattern.startsWith(`**${pathSep}`)) {
-          return normalizedPath.includes(normalizedPattern.slice(3));
-        }
-        return normalizedPath.includes(normalizedPattern);
-      });
-
-    const buildStructure = (dir, depth = 0, prefix = '') => {
-      if (depth > maxDepth) return '';
-      let result = '';
-      try {
-        const entries = fs
-          .readdirSync(dir, { withFileTypes: true })
-          .sort((a, b) => (a.isDirectory() === b.isDirectory() ? a.name.localeCompare(b.name) : a.isDirectory() ? -1 : 1));
-        for (let i = 0; i < entries.length; i++) {
-          const entry = entries[i];
-          const isLast = i === entries.length - 1;
-          const entryPath = path.join(dir, entry.name);
-          const relativePath = path.relative(rootDir, entryPath);
-          // Skip if ignored
-          if (shouldIgnore(relativePath) || entry.name === LANCEDB_DIR_NAME || entry.name === FASTEMBED_CACHE_DIR_NAME) continue; // Also ignore DB/cache dirs
-
-          const connector = isLast ? '└── ' : '├── ';
-          const nextPrefix = isLast ? prefix + '    ' : prefix + '│   ';
-          if (entry.isDirectory()) {
-            result += `${prefix}${connector}${entry.name}/\n`;
-            result += buildStructure(entryPath, depth + 1, nextPrefix);
-          } else if (showFiles) {
-            result += `${prefix}${connector}${entry.name}\n`;
-          }
-        }
-      } catch (error) {
-        console.error(`Error reading directory ${dir}:`, error.message);
-      }
-      return result;
-    };
-    return buildStructure(rootDir);
-  }
-
-  /**
-   * Generate and store an embedding for the project directory structure
-   * @param {Object} options - Options for generating the directory structure
-   * @returns {Promise<boolean>} True if successful, false otherwise
-   */
-  async generateDirectoryStructureEmbedding(options = {}) {
-    console.log(chalk.cyan('[generateDirEmb] Starting...')); // Log entry
-
-    if (!this.modelManager) {
-      throw createFileProcessingError('ModelManager is required for directory structure embedding');
-    }
-
-    if (!this.databaseManager) {
-      throw createFileProcessingError('DatabaseManager is required for directory structure embedding');
-    }
-
-    try {
-      await this.databaseManager.getDB();
-      const table = await this.databaseManager.getTable(this.fileEmbeddingsTable);
-      if (!table) {
-        throw new Error(`[generateDirEmb] Table ${this.fileEmbeddingsTable} not found.`);
-      }
-
-      // Create project-specific structure ID based on the root directory
-      const rootDir = options.rootDir || process.cwd();
-      const projectName = path.basename(path.resolve(rootDir));
-      const structureId = `__project_structure__${projectName}`;
-
-      try {
-        await table.delete(`id = '${structureId}'`);
-        debug('[generateDirEmb] Deleted existing project structure embedding');
-      } catch (error) {
-        if (!error.message.includes('Record not found') && !error.message.includes('cannot find')) {
-          debug(`[generateDirEmb] Error deleting existing project structure: ${error.message}`);
-        } else {
-          debug('[generateDirEmb] No existing project structure to delete.');
-        }
-      }
-
-      const directoryStructure = this.generateDirectoryStructure(options);
-      if (!directoryStructure) throw new Error('[generateDirEmb] Failed to generate directory structure string');
-      debug('[generateDirEmb] Directory structure string generated.');
-
-      // *** Calculate embedding explicitly ***
-      const embedding = await this.modelManager.calculateEmbedding(directoryStructure);
-
-      if (!embedding) {
-        console.error(chalk.red('[generateDirEmb] Failed to calculate embedding for directory structure.'));
-        return false; // Indicate failure
-      }
-      debug(`[generateDirEmb] Embedding calculated, length: ${embedding.length}`);
-
-      const record = {
-        vector: embedding, // Include calculated embedding
-        id: structureId,
-        content: directoryStructure,
-        type: 'directory-structure',
-        name: `${projectName} Project Structure`,
-        path: `${projectName} Project Structure`, // Project-specific path
-        project_path: path.resolve(rootDir), // Add project path for consistency with new schema
-        language: 'text',
-        content_hash: createHash('md5').update(directoryStructure).digest('hex').substring(0, 8),
-        last_modified: new Date().toISOString(), // Use current timestamp for directory structure
-      };
-
-      debug(`[generateDirEmb] Prepared record: ID=${record.id}, Vector length=${record.vector?.length}`);
-      if (record.vector?.length !== this.modelManager.embeddingDimensions) {
-        console.error(chalk.red(`[generateDirEmb] !!! Vector dimension mismatch before add !!!`));
-        return false; // Don't add invalid record
-      }
-
-      // *** Add record with specific try/catch ***
-      debug('[generateDirEmb] Attempting table.add...');
-      try {
-        await table.add([record]);
-        console.log(chalk.green('[generateDirEmb] Successfully added directory structure embedding.'));
-        return true; // Indicate success
-      } catch (addError) {
-        console.error(chalk.red(`[generateDirEmb] !!! Error during table.add: ${addError.message}`), addError.stack);
-        return false; // Indicate failure
-      }
-    } catch (error) {
-      console.error(chalk.red(`[generateDirEmb] Overall error: ${error.message}`), error.stack);
-      return false; // Indicate failure
-    }
-  }
-
-  // ============================================================================
-  // BATCH PROCESSING
-  // ============================================================================
-
-  /**
-   * Process embeddings for multiple files in batch
-   * @param {string[]} filePaths - Array of file paths to process
-   * @param {Object} options - Processing options
-   * @returns {Promise<Object>} Processing results
-   */
-  async processBatchEmbeddings(filePaths, options = {}) {
-    const {
-      excludePatterns = [],
-      respectGitignore = true,
-      baseDir: optionBaseDir = process.cwd(),
-      maxLines = 1000,
-      onProgress, // <<< Add onProgress here
-    } = options;
-    const resolvedCanonicalBaseDir = path.resolve(optionBaseDir);
-    debug(`Resolved canonical base directory: ${resolvedCanonicalBaseDir}`);
-
-    if (!this.modelManager) {
-      throw createFileProcessingError('ModelManager is required for batch processing');
-    }
-
-    if (!this.databaseManager) {
-      throw createFileProcessingError('DatabaseManager is required for batch processing');
-    }
-
-    try {
-      await this.modelManager.initialize(); // Ensure model is ready
-    } catch {
-      console.error(chalk.red('Failed to initialize embedding model. Aborting batch process.'));
-      return { processed: 0, failed: filePaths.length, skipped: 0, excluded: 0, files: [], failedFiles: [...filePaths], excludedFiles: [] };
-    }
-
-    console.log(chalk.blue('Ensuring database tables exist before batch processing...'));
-    try {
-      await this.databaseManager.getDB();
-      console.log(chalk.green('Database table check complete.'));
-    } catch (dbError) {
-      console.error(chalk.red(`Failed to initialize database or tables: ${dbError.message}. Aborting batch process.`));
-      return { processed: 0, failed: filePaths.length, skipped: 0, excluded: 0, files: [], failedFiles: [...filePaths], excludedFiles: [] };
-    }
-
-    const results = { processed: 0, failed: 0, skipped: 0, excluded: 0, files: [], failedFiles: [], excludedFiles: [] };
-    const exclusionOptions = { excludePatterns, respectGitignore, baseDir: resolvedCanonicalBaseDir };
-    this.processedFiles.clear();
-    this.progressTracker.reset(filePaths.length);
-    console.log(chalk.blue(`Starting batch processing of ${filePaths.length} files...`));
-
-    // Generate directory structure embedding first
-    try {
-      await this.generateDirectoryStructureEmbedding({
-        rootDir: resolvedCanonicalBaseDir,
-        maxDepth: 5,
-        ignorePatterns: excludePatterns,
-        showFiles: true,
-      });
-    } catch (structureError) {
-      console.warn(chalk.yellow(`Warning: Failed to generate directory structure embedding: ${structureError.message}`));
-    }
-
-    const fileTable = await this.databaseManager.getTable(this.fileEmbeddingsTable);
-    if (!fileTable) {
-      console.error(chalk.red(`Table ${this.fileEmbeddingsTable} not found. Aborting batch file embedding.`));
-      results.failed = filePaths.length;
-      results.failedFiles = [...filePaths];
-      this.progressTracker.failedCount = filePaths.length;
-      this.progressTracker.update('failed');
-      return results;
-    }
-
-    // Process files in batches
-    console.log(chalk.cyan('--- Starting Phase 1: File Embeddings ---'));
-    const BATCH_SIZE = 50; // Process files in smaller batches for better performance
-
-    for (let i = 0; i < filePaths.length; i += BATCH_SIZE) {
-      const batch = filePaths.slice(i, i + BATCH_SIZE);
-      const batchResults = await this._processBatch(batch, resolvedCanonicalBaseDir, exclusionOptions, onProgress, maxLines);
-
-      // Merge results
-      results.processed += batchResults.processed;
-      results.failed += batchResults.failed;
-      results.skipped += batchResults.skipped;
-      results.excluded += batchResults.excluded;
-      results.files.push(...batchResults.files);
-      results.failedFiles.push(...batchResults.failedFiles);
-      results.excludedFiles.push(...batchResults.excludedFiles);
-    }
-
-    // Process document chunks
-    await this._processDocumentChunks(filePaths, resolvedCanonicalBaseDir, excludePatterns);
-
-    console.log(chalk.green(`Batch processing complete!`));
-
-    // Update progress tracker counts for internal tracking
-    this.progressTracker.processedCount = results.processed;
-    this.progressTracker.skippedCount = results.excluded + results.skipped;
-    this.progressTracker.failedCount = results.failed;
-
-    return results;
-  }
-
-  // ============================================================================
-  // PRIVATE METHODS
-  // ============================================================================
-
-  /**
-   * Process a batch of files
-   * @param {string[]} filePaths - File paths to process
-   * @param {string} baseDir - Base directory
-   * @param {Object} exclusionOptions - Exclusion options
-   * @param {Function} onProgress - Progress callback
-   * @returns {Promise<Object>} Batch processing results
-   * @private
-   */
-  async _processBatch(filePaths, baseDir, exclusionOptions, onProgress, maxLines = 1000) {
-    const results = { processed: 0, failed: 0, skipped: 0, excluded: 0, files: [], failedFiles: [], excludedFiles: [] };
-
-    // ============================================================================
-    // PHASE 1: BATCH GITIGNORE CHECK
-    // ============================================================================
-    let gitignoreCache = new Map();
-    if (exclusionOptions.respectGitignore !== false) {
-      console.log(chalk.cyan(`Performing batch gitignore check for ${filePaths.length} files...`));
-      const gitStartTime = Date.now();
-      const absoluteFilePaths = filePaths.map((fp) => (path.isAbsolute(fp) ? path.resolve(fp) : path.resolve(baseDir, fp)));
-      gitignoreCache = await batchCheckGitignore(absoluteFilePaths, baseDir);
-      const gitDuration = ((Date.now() - gitStartTime) / 1000).toFixed(2);
-      console.log(chalk.green(`✓ Batch gitignore check completed in ${gitDuration}s`));
-    }
-
-    // ============================================================================
-    // PHASE 2: GET EXISTING EMBEDDINGS (for early filtering)
-    // ============================================================================
-    // Query existing embeddings BEFORE reading any files
-    const fileTable = await this.databaseManager.getTable(this.fileEmbeddingsTable);
-    let existingFilesMap = new Map();
-
-    try {
-      const existingRecords = await fileTable
-        .query()
-        .where(`project_path = '${baseDir.replace(/'/g, "''")}'`)
-        .toArray();
-
-      for (const record of existingRecords) {
-        if (!existingFilesMap.has(record.path)) {
-          existingFilesMap.set(record.path, []);
-        }
-        existingFilesMap.get(record.path).push(record);
-      }
-
-      console.log(chalk.cyan(`Found ${existingRecords.length} existing embeddings for comparison`));
-    } catch (queryError) {
-      console.warn(chalk.yellow(`Warning: Could not query existing embeddings: ${queryError.message}`));
-    }
-
-    // ============================================================================
-    // PHASE 3: FAST PRE-FILTERING (without reading file contents)
-    // ============================================================================
-    // Filter files based on basic checks and file timestamps before reading content
-    const candidateFiles = [];
-
-    for (const filePath of filePaths) {
-      const absoluteFilePath = path.isAbsolute(filePath) ? path.resolve(filePath) : path.resolve(baseDir, filePath);
-      const consistentRelativePath = path.relative(baseDir, absoluteFilePath);
-
-      try {
-        // Get file stats (size, mtime, etc.)
-        const stats = fs.statSync(absoluteFilePath);
-
-        // Check if file should be processed (using cached gitignore results)
-        if (
-          !utilsShouldProcessFile(absoluteFilePath, '', {
-            ...exclusionOptions,
-            baseDir: baseDir,
-            relativePathToCheck: consistentRelativePath,
-            gitignoreCache, // Pass the pre-computed cache
-            fileStats: stats, // Pass stats to avoid re-reading
-          })
-        ) {
-          results.excluded++;
-          results.excludedFiles.push(filePath);
-          this.progressTracker.update('skipped');
-          if (typeof onProgress === 'function') onProgress('excluded', filePath);
-          this.processedFiles.set(filePath, 'excluded');
-          continue;
-        }
-
-        // Early skip based on modification time if file exists in database
-        const existingRecords = existingFilesMap.get(consistentRelativePath) || [];
-        let potentiallyUnchanged = false;
-
-        if (existingRecords.length > 0) {
-          // Check if any existing record has the same modification time
-          for (const existing of existingRecords) {
-            const existingMtime = new Date(existing.last_modified).getTime();
-            const currentMtime = stats.mtime.getTime();
-
-            // If modification times match (within 1 second to account for filesystem precision)
-            if (Math.abs(existingMtime - currentMtime) < 1000) {
-              potentiallyUnchanged = true;
-              break;
-            }
-          }
-        }
-
-        candidateFiles.push({
-          filePath: absoluteFilePath,
-          originalInputPath: filePath,
-          relativePath: consistentRelativePath,
-          stats,
-          potentiallyUnchanged,
-          existingRecords,
-        });
-      } catch {
-        results.failed++;
-        results.failedFiles.push(filePath);
-        this.progressTracker.update('failed');
-        if (typeof onProgress === 'function') onProgress('failed', filePath);
-        this.processedFiles.set(filePath, 'failed_stat');
-      }
-    }
-
-    console.log(chalk.cyan(`Pre-filtered to ${candidateFiles.length} candidate files (excluded ${results.excluded})`));
-
-    // ============================================================================
-    // PHASE 4: READ FILES AND CONTENT HASH CHECK
-    // ============================================================================
-    // Now read file contents only for candidates that passed initial filtering
-    const filesToProcess = [];
-    const contentsForBatch = [];
-
-    for (const fileData of candidateFiles) {
-      try {
-        // Read file content
-        let content = await fs.promises.readFile(fileData.filePath, 'utf8');
-
-        // Check if empty
-        if (content.trim().length === 0) {
-          results.skipped++;
-          this.progressTracker.update('skipped');
-          if (typeof onProgress === 'function') onProgress('skipped', fileData.originalInputPath);
-          this.processedFiles.set(fileData.originalInputPath, 'skipped_empty');
-          continue;
-        }
-
-        // Truncate content to maximum specified lines for code files only
-        const isDocFile = isDocumentationFile(fileData.filePath);
-        if (!isDocFile) {
-          const lines = content.split('\n');
-          if (lines.length > maxLines) {
-            content = lines.slice(0, maxLines).join('\n') + '\n... (truncated from ' + lines.length + ' lines)';
-            debug(`Truncated code file ${fileData.relativePath} from ${lines.length} lines to ${maxLines} lines`);
-          }
-        }
-
-        // Add content to file data
-        fileData.content = content;
-        filesToProcess.push(fileData);
-        contentsForBatch.push(content);
-      } catch {
-        results.failed++;
-        results.failedFiles.push(fileData.originalInputPath);
-        this.progressTracker.update('failed');
-        if (typeof onProgress === 'function') onProgress('failed', fileData.originalInputPath);
-        this.processedFiles.set(fileData.originalInputPath, 'failed_read');
-      }
-    }
-
-    // ============================================================================
-    // PHASE 5: CONTENT HASH CHECK AND DEDUPLICATION
-    // ============================================================================
-    // Check each file against existing embeddings using content hash
-    const filesToActuallyProcess = [];
-    const contentsToActuallyProcess = [];
-    const recordsToDelete = [];
-
-    for (let i = 0; i < filesToProcess.length; i++) {
-      const fileData = filesToProcess[i];
-      const contentHash = createHash('md5').update(fileData.content).digest('hex').substring(0, 8);
-
-      const existingRecords = fileData.existingRecords || [];
-      let needsUpdate = true;
-
-      if (existingRecords.length > 0) {
-        // Check if any existing record matches our current file state
-        for (const existing of existingRecords) {
-          if (existing.content_hash === contentHash) {
-            // File content hasn't changed - skip processing (CI-friendly)
-            // Note: We rely on content_hash rather than last_modified because
-            // GitHub Actions checkout changes file timestamps even for unchanged files
-            needsUpdate = false;
-            results.skipped++;
-            this.progressTracker.update('skipped');
-            if (typeof onProgress === 'function') onProgress('skipped', fileData.originalInputPath);
-            this.processedFiles.set(fileData.originalInputPath, 'skipped_unchanged');
-            debug(`Skipping unchanged file: ${fileData.relativePath} (hash: ${contentHash})`);
-            break;
-          } else if (existing.path === fileData.relativePath) {
-            // Same file path but different content - mark old version for deletion
-            recordsToDelete.push(existing);
-          }
-        }
-      }
-
-      if (needsUpdate) {
-        // File needs processing (new or changed)
-        filesToActuallyProcess.push(fileData);
-        contentsToActuallyProcess.push(fileData.content);
-      }
-    }
-
-    // Batch delete old versions if any
-    if (recordsToDelete.length > 0) {
-      for (const recordToDelete of recordsToDelete) {
-        try {
-          await fileTable.delete(`id = '${recordToDelete.id.replace(/'/g, "''")}'`);
-          debug(`Deleted old version: ${recordToDelete.path} (old hash: ${recordToDelete.content_hash})`);
-        } catch (deleteError) {
-          console.warn(chalk.yellow(`Warning: Could not delete old version of ${recordToDelete.path}: ${deleteError.message}`));
-        }
-      }
-    }
-
-    // Generate embeddings only for files that need processing
-    if (filesToActuallyProcess.length > 0) {
-      console.log(
-        chalk.cyan(
-          `Processing ${filesToActuallyProcess.length} new/changed files (skipped ${filesToProcess.length - filesToActuallyProcess.length} unchanged)`
-        )
-      );
-
-      try {
-        const embeddings = await this.modelManager.calculateEmbeddingBatch(contentsToActuallyProcess);
-        const recordsToAdd = [];
-
-        for (let i = 0; i < embeddings.length; i++) {
-          const fileData = filesToActuallyProcess[i];
-          const embeddingVector = embeddings[i];
-
-          if (embeddingVector) {
-            const contentHash = createHash('md5').update(fileData.content).digest('hex').substring(0, 8);
-            const fileId = `${fileData.relativePath}#${contentHash}`;
-
-            const record = {
-              vector: embeddingVector,
-              id: fileId,
-              content: fileData.content,
-              type: 'file',
-              name: path.basename(fileData.filePath),
-              path: fileData.relativePath,
-              project_path: baseDir,
-              language: detectLanguageFromExtension(path.extname(fileData.filePath)),
-              content_hash: contentHash,
-              last_modified: fileData.stats.mtime.toISOString(),
-            };
-            recordsToAdd.push(record);
-          } else {
-            results.failed++;
-            results.failedFiles.push(fileData.originalInputPath);
-            this.progressTracker.update('failed');
-            if (typeof onProgress === 'function') onProgress('failed', fileData.originalInputPath);
-            this.processedFiles.set(fileData.originalInputPath, 'failed_embedding');
-          }
-        }
-
-        // Add new/updated records to database
-        if (recordsToAdd.length > 0) {
-          await fileTable.add(recordsToAdd);
-
-          // Optimize table to sync indices with data and prevent TakeExec panics
-          try {
-            await fileTable.optimize();
-          } catch (optimizeError) {
-            if (optimizeError.message && optimizeError.message.includes('legacy format')) {
-              console.log(
-                chalk.yellow(`Skipping optimization due to legacy index format - will be auto-upgraded during normal operations`)
-              );
-            } else {
-              console.warn(
-                chalk.yellow(`Warning: Failed to optimize file embeddings table after adding records: ${optimizeError.message}`)
-              );
-            }
-          }
-
-          recordsToAdd.forEach((record, index) => {
-            const fileData = filesToActuallyProcess[index];
-            if (embeddings[index]) {
-              results.processed++;
-              results.files.push(fileData.originalInputPath);
-              this.progressTracker.update('processed');
-              if (typeof onProgress === 'function') onProgress('processed', fileData.originalInputPath);
-              this.processedFiles.set(fileData.originalInputPath, 'processed');
-            }
-          });
-        }
-      } catch (error) {
-        console.error(chalk.red(`Error processing batch: ${error.message}`));
-        filesToProcess.forEach((fileData) => {
-          results.failed++;
-          results.failedFiles.push(fileData.originalInputPath);
-          this.progressTracker.update('failed');
-          if (typeof onProgress === 'function') onProgress('failed', fileData.originalInputPath);
-          this.processedFiles.set(fileData.originalInputPath, 'failed_batch');
-        });
-      }
-    }
-
-    return results;
-  }
-
-  /**
-   * Process document chunks
-   * @param {string[]} filePaths - File paths to process
-   * @param {string} baseDir - Base directory
-   * @param {string[]} excludePatterns - Exclude patterns
-   * @returns {Promise<void>}
-   * @private
-   */
-  async _processDocumentChunks(filePaths, baseDir) {
-    console.log(chalk.cyan('--- Starting Phase 2: Document Chunk Embeddings ---'));
-    const documentChunkTable = await this.databaseManager.getTable(this.documentChunkTable);
-    if (!documentChunkTable) {
-      console.warn(chalk.yellow(`Skipping Phase 2: Document Chunk Embeddings because table ${this.documentChunkTable} was not found.`));
-      return;
-    }
-
-    // Efficient batch check: Get all existing document chunks for this project
-    let existingDocChunksMap = new Map();
-    try {
-      const existingChunks = await documentChunkTable
-        .query()
-        .where(`project_path = '${baseDir.replace(/'/g, "''")}'`)
-        .toArray();
-
-      // Build a map for fast lookup: original_document_path -> [chunks]
-      for (const chunk of existingChunks) {
-        if (!existingDocChunksMap.has(chunk.original_document_path)) {
-          existingDocChunksMap.set(chunk.original_document_path, []);
-        }
-        existingDocChunksMap.get(chunk.original_document_path).push(chunk);
-      }
-
-      console.log(chalk.cyan(`Found ${existingChunks.length} existing document chunks for comparison`));
-    } catch (queryError) {
-      console.warn(chalk.yellow(`Warning: Could not query existing document chunks, will process all docs: ${queryError.message}`));
-      existingDocChunksMap = new Map();
-    }
-
-    const allDocChunksToEmbed = [];
-    const allDocChunkRecordsToAdd = [];
-    const processedDocPathsForDeletion = new Set();
-    let skippedDocCount = 0;
-
-    for (const filePath of filePaths) {
-      const absoluteFilePath = path.isAbsolute(filePath) ? path.resolve(filePath) : path.resolve(baseDir, filePath);
-      const consistentRelativePath = path.relative(baseDir, absoluteFilePath);
-      const language = detectLanguageFromExtension(path.extname(absoluteFilePath));
-
-      if (isDocumentationFile(absoluteFilePath, language)) {
-        try {
-          const stats = fs.statSync(absoluteFilePath);
-          if (stats.size > 5 * 1024 * 1024) {
-            // 5MB limit for docs
-            continue;
-          }
-
-          const content = await fs.promises.readFile(absoluteFilePath, 'utf8');
-          if (content.trim().length === 0) {
-            continue;
-          }
-
-          // Check if document has changed by comparing chunk content hashes
-          const existingChunks = existingDocChunksMap.get(consistentRelativePath) || [];
-
-          // Extract chunks to compare with existing ones
-          const { chunks: currentChunks, documentH1 } = extractMarkdownChunks(absoluteFilePath, content, consistentRelativePath);
-          let hasUnchangedDocument = false;
-
-          if (existingChunks.length > 0 && currentChunks.length === existingChunks.length) {
-            // Create a signature of the document by combining all chunk content hashes
-            const currentChunkHashes = currentChunks
-              .map((chunk) => createHash('md5').update(chunk.content).digest('hex').substring(0, 8))
-              .sort()
-              .join('|');
-
-            const existingChunkHashes = existingChunks
-              .map((chunk) => chunk.content_hash)
-              .sort()
-              .join('|');
-
-            hasUnchangedDocument = currentChunkHashes === existingChunkHashes;
-          }
-
-          if (hasUnchangedDocument) {
-            // Document hasn't changed - skip processing
-            skippedDocCount++;
-            debug(`Skipping unchanged document: ${consistentRelativePath} (${currentChunks.length} chunks match)`);
-            continue;
-          }
-
-          // Document has changed or is new - process it
-          if (!processedDocPathsForDeletion.has(consistentRelativePath)) {
-            processedDocPathsForDeletion.add(consistentRelativePath);
-          }
-
-          if (currentChunks.length > 0) {
-            currentChunks.forEach((chunk) => {
-              const chunkWithTitle = {
-                ...chunk,
-                documentTitle: documentH1 || path.basename(absoluteFilePath, path.extname(absoluteFilePath)),
-                fileStats: stats,
-              };
-              allDocChunksToEmbed.push(chunkWithTitle);
-            });
-          }
-        } catch (docError) {
-          console.warn(chalk.yellow(`Error processing document ${consistentRelativePath} for chunking: ${docError.message}`));
-        }
-      }
-    }
-
-    if (skippedDocCount > 0) {
-      console.log(chalk.cyan(`Skipped ${skippedDocCount} unchanged documentation files`));
-    }
-
-    if (allDocChunksToEmbed.length > 0) {
-      console.log(chalk.blue(`Extracted ${allDocChunksToEmbed.length} total document chunks to process for embeddings.`));
-      const chunkContentsForBatching = allDocChunksToEmbed.map((chunk) => chunk.content);
-      const chunkEmbeddings = await this.modelManager.calculateEmbeddingBatch(chunkContentsForBatching);
-
-      for (let i = 0; i < chunkEmbeddings.length; i++) {
-        const chunkData = allDocChunksToEmbed[i];
-        const chunkEmbeddingVector = chunkEmbeddings[i];
-
-        if (chunkEmbeddingVector) {
-          const chunkContentHash = createHash('md5').update(chunkData.content).digest('hex').substring(0, 8);
-          const chunkId = `${chunkData.original_document_path}#${slugify(chunkData.heading || 'section')}_${chunkData.start_line_in_doc}`;
-
-          const record = {
-            id: chunkId,
-            content: chunkData.content,
-            original_document_path: chunkData.original_document_path,
-            project_path: baseDir,
-            heading_text: chunkData.heading || '',
-            document_title: chunkData.documentTitle,
-            language: chunkData.language || 'markdown',
-            vector: chunkEmbeddingVector,
-            content_hash: chunkContentHash,
-            last_modified: chunkData.fileStats ? chunkData.fileStats.mtime.toISOString() : new Date().toISOString(),
-          };
-          allDocChunkRecordsToAdd.push(record);
-        }
-      }
-    }
-
-    // Delete old chunks and add new ones
-    if (processedDocPathsForDeletion.size > 0) {
-      for (const docPathToDelete of processedDocPathsForDeletion) {
-        try {
-          await documentChunkTable.delete(`original_document_path = '${docPathToDelete.replace(/'/g, "''")}'`);
-        } catch (deleteError) {
-          console.warn(chalk.yellow(`Error deleting chunks for document ${docPathToDelete}: ${deleteError.message}`));
-        }
-      }
-    }
-
-    if (allDocChunkRecordsToAdd.length > 0) {
-      try {
-        await documentChunkTable.add(allDocChunkRecordsToAdd);
-
-        // Optimize table to sync indices with data and prevent TakeExec panics
-        try {
-          await documentChunkTable.optimize();
-        } catch (optimizeError) {
-          if (optimizeError.message && optimizeError.message.includes('legacy format')) {
-            console.log(chalk.yellow(`Skipping optimization due to legacy index format - will be auto-upgraded during normal operations`));
-          } else {
-            console.warn(chalk.yellow(`Warning: Failed to optimize document chunk table after adding records: ${optimizeError.message}`));
-          }
-        }
-
-        console.log(
-          chalk.green(`Successfully added ${allDocChunkRecordsToAdd.length} document chunk embeddings to ${this.documentChunkTable}.`)
-        );
-      } catch (addError) {
-        console.error(chalk.red(`Error batch adding document chunk embeddings to DB: ${addError.message}`), addError.stack);
-      }
-    }
-
-    console.log(chalk.green('--- Finished Phase 2: Document Chunk Embeddings ---'));
-  }
-
-  // ============================================================================
-  // CLEANUP
-  // ============================================================================
-
-  /**
-   * Clean up file processor resources
-   */
-  async cleanup() {
-    if (this.cleaningUp) {
-      return; // Already cleaning up, prevent duplicate calls
-    }
-
-    this.cleaningUp = true;
-
-    try {
-      this.processedFiles.clear();
-      this.progressTracker.reset(0);
-      console.log(chalk.green('[FileProcessor] Resources cleaned up.'));
-    } catch (error) {
-      console.error(chalk.red(`[FileProcessor] Error during cleanup: ${error.message}`));
-    } finally {
-      this.cleaningUp = false;
-    }
-  }
-}