codecritique 1.0.0 → 1.1.0

package/src/embeddings/model-manager.js

@@ -1,337 +0,0 @@
-/**
- * Model Manager Module
- *
- * This module provides centralized embedding model management using FastEmbed.
- * It handles model initialization, caching, and all embedding generation operations.
- *
- * Features:
- * - Singleton model instance management
- * - Thread-safe model initialization
- * - Embedding generation with caching
- * - Batch embedding processing
- * - Query-specific embedding generation
- * - Comprehensive error handling
- */
-/**
- * @typedef {import('./types.js').EmbeddingVector} EmbeddingVector
- * @typedef {import('./types.js').QueryEmbeddingOptions} QueryEmbeddingOptions
- * @typedef {import('./types.js').BatchProcessingOptions} BatchProcessingOptions
- */
-
-import fs from 'node:fs';
-import chalk from 'chalk';
-import dotenv from 'dotenv';
-import { EmbeddingModel, FlagEmbedding } from 'fastembed';
-import { debug } from '../utils/logging.js';
-import { EMBEDDING_DIMENSIONS, MODEL_NAME_STRING, MAX_RETRIES } from './constants.js';
-import { FASTEMBED_CACHE_DIR } from './constants.js';
-import { createModelInitializationError, createEmbeddingGenerationError } from './errors.js';
-
-// Load environment variables
-dotenv.config();
-
-// ============================================================================
-// MODEL MANAGER CLASS
-// ============================================================================
-
-export class ModelManager {
-  constructor(options = {}) {
-    this.embeddingDimensions = options.embeddingDimensions || EMBEDDING_DIMENSIONS;
-    this.modelNameString = options.modelNameString || MODEL_NAME_STRING;
-    this.maxRetries = options.maxRetries || MAX_RETRIES;
-    this.cacheDir = options.cacheDir || FASTEMBED_CACHE_DIR;
-    this.cacheManager = options.cacheManager || null;
-
-    // Model state
-    this.embeddingModel = null;
-    this.modelInitialized = false;
-    this.modelInitializationPromise = null;
-    this.cleaningUp = false;
-
-    console.log(chalk.magenta(`[ModelManager] Using MODEL = ${this.modelNameString}, DIMENSIONS = ${this.embeddingDimensions}`));
-  }
-
-  // ============================================================================
-  // MODEL INITIALIZATION
-  // ============================================================================
-
-  /**
-   * Initialize the FastEmbed model instance
-   * @returns {Promise<import('fastembed').FlagEmbedding>} Initialized model instance
-   */
-  async initialize() {
-    // If model is already initialized, return it immediately
-    if (this.embeddingModel) {
-      return this.embeddingModel;
-    }
-
-    // If initialization is already in progress, wait for it
-    if (this.modelInitializationPromise) {
-      return await this.modelInitializationPromise;
-    }
-
-    // Start initialization and store the promise
-    this.modelInitializationPromise = (async () => {
-      const modelIdentifier = EmbeddingModel.BGESmallENV15;
-
-      // Only print logs if we haven't initialized before
-      if (!this.modelInitialized) {
-        console.log(chalk.blue(`Attempting to initialize fastembed model. Identifier: ${this.modelNameString}`));
-        console.log(chalk.blue(`FastEmbed Cache Directory: ${this.cacheDir}`));
-      }
-
-      try {
-        if (!fs.existsSync(this.cacheDir)) {
-          console.log(chalk.yellow(`Creating fastembed cache directory: ${this.cacheDir}`));
-          fs.mkdirSync(this.cacheDir, { recursive: true });
-        }
-
-        let retries = 0;
-        while (retries < this.maxRetries) {
-          try {
-            this.embeddingModel = await FlagEmbedding.init({
-              model: modelIdentifier,
-              cacheDir: this.cacheDir,
-            });
-
-            // Only print success message if we haven't initialized before
-            if (!this.modelInitialized) {
-              console.log(chalk.green('FastEmbed model initialized successfully.'));
-              this.modelInitialized = true;
-            }
-            break; // Exit loop on success
-          } catch (initError) {
-            retries++;
-            console.error(chalk.yellow(`Model initialization attempt ${retries}/${this.maxRetries} failed: ${initError.message}`));
-            if (retries >= this.maxRetries) {
-              throw createModelInitializationError(
-                `Failed to initialize model after ${this.maxRetries} attempts: ${initError.message}`,
-                initError,
-                { modelIdentifier, cacheDir: this.cacheDir }
-              );
-            }
-            await new Promise((resolve) => setTimeout(resolve, retries * 2000)); // Wait before retrying
-          }
-        }
-
-        // Clear the initialization promise since we're done
-        this.modelInitializationPromise = null;
-        return this.embeddingModel;
-      } catch (err) {
-        // Clear the initialization promise on error
-        this.modelInitializationPromise = null;
-        console.error(chalk.red(`Fatal: Failed to initialize fastembed model: ${err.message}`), err);
-        throw err; // Re-throw critical error
-      }
-    })();
-
-    return await this.modelInitializationPromise;
-  }
-
-  /**
-   * Check if the model is initialized
-   * @returns {boolean} True if model is initialized
-   */
-  isInitialized() {
-    return this.modelInitialized && this.embeddingModel !== null;
-  }
-
-  // ============================================================================
-  // EMBEDDING GENERATION
-  // ============================================================================
-
-  /**
-   * Calculate embedding for a text using fastembed
-   * @param {string} text - The text to embed
-   * @returns {Promise<EmbeddingVector|null>} The embedding vector or null on error
-   */
-  async calculateEmbedding(text) {
-    // Ensure text is a non-empty string
-    if (typeof text !== 'string' || text.trim().length === 0) {
-      return null; // Return null for empty text to avoid errors downstream
-    }
-
-    // Check cache first
-    const cacheKey = text.trim().substring(0, 200); // Use first 200 chars as cache key
-    if (this.cacheManager) {
-      const cachedResult = this.cacheManager.getEmbedding(cacheKey);
-      if (cachedResult) {
-        return cachedResult;
-      }
-    }
-
-    try {
-      const model = await this.initialize();
-      let embedding = null; // Initialize embedding as null
-
-      // Use passageEmbed which is suitable for sentences/paragraphs/code snippets
-      const embeddingGenerator = model.passageEmbed([text]);
-      // FastEmbed's async generator yields batches, even for single input
-      for await (const batch of embeddingGenerator) {
-        if (batch && batch.length > 0 && batch[0]) {
-          embedding = Array.from(batch[0]); // Convert Float32Array to regular array
-          break; // Got the embedding for the single input text
-        }
-      }
-
-      // Validate the generated embedding
-      if (!embedding || !Array.isArray(embedding) || embedding.length !== this.embeddingDimensions) {
-        console.error(
-          chalk.red(
-            `Generated embedding dimension (${embedding?.length}) does not match expected (${this.embeddingDimensions}) or embedding is invalid.`
-          )
-        );
-        return null; // Return null if dimensions mismatch or invalid
-      }
-
-      // Cache the result
-      if (this.cacheManager) {
-        this.cacheManager.setEmbedding(cacheKey, embedding);
-      }
-
-      return embedding;
-    } catch (error) {
-      console.error(chalk.red(`Error calculating embedding: ${error.message}`), error);
-      throw createEmbeddingGenerationError(`Failed to calculate embedding: ${error.message}`, error, { text: text.substring(0, 100) });
-    }
-  }
-
-  /**
-   * Calculate embeddings for a batch of texts using fastembed
-   * @param {string[]} texts - An array of texts to embed
-   * @param {BatchProcessingOptions} [options] - Batch processing options
-   * @returns {Promise<Array<EmbeddingVector|null>>} A promise that resolves to an array of embedding vectors
-   */
-  async calculateEmbeddingBatch(texts) {
-    // Ensure texts is a non-empty array of non-empty strings
-    if (!Array.isArray(texts) || texts.length === 0 || texts.some((text) => typeof text !== 'string' || text.trim().length === 0)) {
-      debug('Skipping batch embedding for empty or invalid texts array.');
-      // Return an array of nulls corresponding to the input, or an empty array if appropriate
-      return texts.map(() => null);
-    }
-
-    try {
-      const model = await this.initialize();
-      const embeddings = [];
-
-      // passageEmbed is an async generator of batches
-      for await (const batch of model.passageEmbed(texts)) {
-        for (const vec of batch) {
-          // Validate each generated embedding
-          if (vec && typeof vec.length === 'number' && vec.length === this.embeddingDimensions) {
-            embeddings.push(Array.from(vec)); // Convert Float32Array (or other array-like) to regular array
-          } else {
-            console.error(
-              chalk.red(
-                `Generated batch embedding dimension (${vec?.length}) does not match expected (${this.embeddingDimensions}) or embedding is invalid.`
-              )
-            );
-            embeddings.push(null); // Add null for invalid embeddings in the batch
-          }
-        }
-      }
-
-      // Ensure the number of embeddings matches the number of input texts
-      if (embeddings.length !== texts.length) {
-        console.error(
-          chalk.red(`Number of generated embeddings (${embeddings.length}) does not match number of input texts (${texts.length}).`)
-        );
-        // This case should ideally be handled by ensuring one embedding (or null) per input text.
-        // For now, if there's a mismatch, it might indicate a deeper issue.
-        // We'll return what we have, but this could lead to misaligned data.
-      }
-
-      debug(`Batch embeddings generated successfully, count: ${embeddings.filter((e) => e !== null).length}`);
-      return embeddings;
-    } catch (error) {
-      console.error(chalk.red(`Error calculating batch embeddings: ${error.message}`), error);
-      throw createEmbeddingGenerationError(`Failed to calculate batch embeddings: ${error.message}`, error, { textsCount: texts.length });
-    }
-  }
-
-  /**
-   * Calculate embedding for a query text using fastembed
-   * @param {string} text - The query text to embed
-   * @param {QueryEmbeddingOptions} [options] - Query embedding options
-   * @returns {Promise<EmbeddingVector|null>} The embedding vector or null on error
-   */
-  async calculateQueryEmbedding(text) {
-    if (typeof text !== 'string' || text.trim().length === 0) {
-      return null;
-    }
-
-    // Check cache first (use 'query:' prefix to distinguish from passage embeddings)
-    const cacheKey = `query:${text.trim().substring(0, 200)}`;
-    if (this.cacheManager) {
-      const cachedResult = this.cacheManager.getEmbedding(cacheKey);
-      if (cachedResult) {
-        return cachedResult;
-      }
-    }
-
-    try {
-      const model = await this.initialize();
-      // queryEmbed directly returns the embedding for the single query text
-      const embeddingArray = await model.queryEmbed(text);
-
-      // Validate the generated query embedding
-      if (embeddingArray && typeof embeddingArray.length === 'number' && embeddingArray.length === this.embeddingDimensions) {
-        // queryEmbed in fastembed-js v0.2.0+ might return number[] directly or Float32Array
-        // Array.from() handles both cases correctly, converting Float32Array to number[] or returning number[] as is.
-        const embedding = Array.from(embeddingArray);
-
-        // Cache the result
-        if (this.cacheManager) {
-          this.cacheManager.setEmbedding(cacheKey, embedding);
-        }
-
-        return embedding;
-      } else {
-        console.error(
-          chalk.red(
-            `Generated query embedding dimension (${embeddingArray?.length}) does not match expected (${this.embeddingDimensions}) or embedding is invalid.`
-          )
-        );
-        return null;
-      }
-    } catch (error) {
-      console.error(chalk.red(`Error calculating query embedding: ${error.message}`), error);
-      throw createEmbeddingGenerationError(`Failed to calculate query embedding: ${error.message}`, error, {
-        text: text.substring(0, 100),
-      });
-    }
-  }
-
-  // ============================================================================
-  // CLEANUP
-  // ============================================================================
-
-  /**
-   * Clean up model resources
-   */
-  async cleanup() {
-    if (this.cleaningUp) {
-      return; // Already cleaning up, prevent duplicate calls
-    }
-
-    this.cleaningUp = true;
-
-    try {
-      // FastEmbed models don't have an explicit cleanup method
-      // but we can clear our references
-      this.embeddingModel = null;
-      this.modelInitialized = false;
-      this.modelInitializationPromise = null;
-
-      if (this.cacheManager) {
-        this.cacheManager.clearCache('embedding');
-      }
-
-      console.log(chalk.green('[ModelManager] Model resources cleaned up.'));
-    } catch (error) {
-      console.error(chalk.red(`[ModelManager] Error during cleanup: ${error.message}`));
-    } finally {
-      this.cleaningUp = false;
-    }
-  }
-}

package/src/embeddings/similarity-calculator.js

@@ -1,97 +0,0 @@
-/**
- * Similarity Calculator
- *
- * This module contains pure mathematical functions for calculating similarity between vectors,
- * paths, and other data structures. These functions have no external dependencies and are
- * safe to extract for modular use.
- */
-
-import path from 'node:path';
-import { debug } from '../utils/logging.js';
-
-/**
- * Calculate cosine similarity between two vectors
- *
- * @param {Array<number>} vecA - First vector
- * @param {Array<number>} vecB - Second vector
- * @returns {number} Cosine similarity score between -1 and 1
- */
-export function calculateCosineSimilarity(vecA, vecB) {
-  if (!vecA || !vecB || !Array.isArray(vecA) || !Array.isArray(vecB) || vecA.length !== vecB.length || vecA.length === 0) {
-    // Add more robust checks
-    debug(`Invalid input for cosine similarity: vecA length=${vecA?.length}, vecB length=${vecB?.length}`);
-    return 0;
-  }
-
-  let dotProduct = 0;
-  let normA = 0;
-  let normB = 0;
-
-  const len = vecA.length; // Cache length
-
-  for (let i = 0; i < len; i++) {
-    const a = vecA[i]; // Cache values
-    const b = vecB[i];
-    dotProduct += a * b;
-    normA += a * a;
-    normB += b * b;
-  }
-
-  // Check for zero vectors, handle potential floating point inaccuracies
-  if (normA <= 1e-9 || normB <= 1e-9) {
-    return 0;
-  }
-
-  // Clamp result to handle potential floating point errors leading to > 1 or < -1
-  return Math.max(-1.0, Math.min(1.0, dotProduct / (Math.sqrt(normA) * Math.sqrt(normB))));
-}
-
-/**
- * Calculate path similarity between two file paths
- *
- * This function compares two file paths and returns a similarity score based on
- * the common directory prefix. The score is normalized between 0 and 1.
- *
- * @param {string} path1 - First file path
- * @param {string} path2 - Second file path
- * @returns {number} Similarity score between 0 and 1
- */
-export function calculatePathSimilarity(path1, path2) {
-  if (!path1 || !path2) return 0;
-
-  try {
-    // Normalize paths and split into directory components
-    const parts1 = path
-      .dirname(path.normalize(path1))
-      .split(path.sep)
-      .filter((p) => p);
-    const parts2 = path
-      .dirname(path.normalize(path2))
-      .split(path.sep)
-      .filter((p) => p);
-
-    let commonPrefixLength = 0;
-    const minLength = Math.min(parts1.length, parts2.length);
-
-    for (let i = 0; i < minLength; i++) {
-      if (parts1[i] === parts2[i]) {
-        commonPrefixLength++;
-      } else {
-        break;
-      }
-    }
-
-    // Calculate score: common prefix length relative to the average length
-    // Avoid division by zero
-    const avgLength = (parts1.length + parts2.length) / 2;
-    if (avgLength === 0) {
-      return 1; // Both paths are likely in the root or identical
-    }
-
-    const score = commonPrefixLength / avgLength;
-    return Math.max(0, Math.min(1, score)); // Clamp score between 0 and 1
-  } catch (error) {
-    debug(`[calculatePathSimilarity] Error comparing paths '${path1}' and '${path2}': ${error.message}`);
-    return 0; // Return 0 similarity on error
-  }
-}

package/src/embeddings/types.js

@@ -1,113 +0,0 @@
-/**
- * Type Definitions for Embeddings System
- *
- * This module provides TypeScript-style interfaces and type definitions
- * for the embeddings system. These help with documentation and development
- * even in a JavaScript environment.
- */
-
-/**
- * @typedef {Object} EmbeddingVector
- * @property {number[]} vector - The embedding vector array
- * @property {number} dimensions - Number of dimensions in the vector
- * @property {string} model - Model used to generate the embedding
- * @property {string} [id] - Optional identifier for the embedding
- */
-
-/**
- * @typedef {Object} SearchResult
- * @property {string} content - The content that was found
- * @property {string} path - File path of the content
- * @property {number} similarity - Similarity score (0-1)
- * @property {string} [language] - Programming language of the content
- * @property {string} [context] - Additional context information
- * @property {number} [line_start] - Starting line number
- * @property {number} [line_end] - Ending line number
- * @property {boolean} [reranked] - Whether the result has been reranked
- * @property {Object} [metadata] - Additional metadata
- */
-
-/**
- * @typedef {Object} SearchOptions
- * @property {number} [limit] - Maximum number of results to return
- * @property {number} [threshold] - Minimum similarity threshold
- * @property {string} [language] - Filter by programming language
- * @property {string} [path] - Filter by file path pattern
- * @property {boolean} [includeMetadata] - Include metadata in results
- * @property {boolean} [rerank] - Whether to rerank results
- * @property {string} [context] - Additional context for search
- */
-
-/**
- * @typedef {Object} DocumentChunk
- * @property {string} content - The text content of the chunk
- * @property {string} document_title - Title of the document
- * @property {string} document_path - Path to the document
- * @property {number} chunk_index - Index of the chunk within the document
- * @property {number[]} embedding - Embedding vector for the chunk
- * @property {string} [h1_title] - H1 title if applicable
- * @property {number[]} [h1_embedding] - H1 embedding if applicable
- * @property {string} [language] - Programming language
- * @property {Object} [metadata] - Additional metadata
- */
-
-/**
- * @typedef {Object} EmbeddingConfig
- * @property {string} modelName - Name of the embedding model
- * @property {number} dimensions - Number of dimensions
- * @property {string} lancedbPath - Path to LanceDB database
- * @property {string} fastembedCacheDir - FastEmbed cache directory
- * @property {number} maxRetries - Maximum number of retries
- * @property {boolean} debug - Enable debug mode
- * @property {number} maxConcurrency - Maximum concurrent operations
- * @property {number} batchSize - Batch size for processing
- */
-
-/**
- * @typedef {Object} CacheMetrics
- * @property {number} hits - Number of cache hits
- * @property {number} misses - Number of cache misses
- * @property {number} size - Current cache size
- * @property {number} maxSize - Maximum cache size
- * @property {number} evictions - Number of evictions
- * @property {number} hitRate - Hit rate percentage
- */
-
-/**
- * @typedef {Object} ProcessingProgress
- * @property {number} totalFiles - Total number of files to process
- * @property {number} processedCount - Number of files processed
- * @property {number} skippedCount - Number of files skipped
- * @property {number} failedCount - Number of files failed
- * @property {number} startTime - Processing start time
- * @property {number} currentTime - Current time
- * @property {number} estimatedTimeRemaining - Estimated time remaining
- * @property {number} percentComplete - Percentage complete
- */
-
-/**
- * @typedef {Object} DatabaseSchema
- * @property {string} tableName - Name of the table
- * @property {Object} fields - Field definitions
- * @property {string[]} indexes - Index definitions
- * @property {string} primaryKey - Primary key field
- */
-
-/**
- * @typedef {Object} BatchProcessingOptions
- * @property {number} batchSize - Size of each batch
- * @property {number} maxConcurrency - Maximum concurrent batches
- * @property {boolean} skipExisting - Skip files that already exist
- * @property {Function} [progressCallback] - Progress callback function
- * @property {Function} [errorCallback] - Error callback function
- */
-
-/**
- * @typedef {Object} QueryEmbeddingOptions
- * @property {string} [context] - Additional context for the query
- * @property {string} [language] - Programming language hint
- * @property {boolean} [normalize] - Whether to normalize the embedding
- * @property {Object} [metadata] - Additional metadata
- */
-
-export {};