claude-self-reflect 3.0.0 → 3.0.1

package/scripts/importer/embeddings/fastembed_provider.py

@@ -0,0 +1,164 @@
+"""FastEmbed provider for local embeddings."""
+
+from typing import List, Any
+import logging
+import statistics
+from .base import EmbeddingProvider
+from ..core.exceptions import EmbeddingError
+
+logger = logging.getLogger(__name__)
+
+
+class FastEmbedProvider(EmbeddingProvider):
+    """
+    FastEmbed provider for generating embeddings locally.
+    
+    Uses sentence-transformers/all-MiniLM-L6-v2 model by default.
+    """
+    
+    def __init__(self):
+        super().__init__()
+        self.model = None
+        self.model_name = None
+        self.dimension = None
+    
+    def initialize(self, config: Any) -> None:
+        """Initialize FastEmbed with the specified model."""
+        try:
+            from fastembed import TextEmbedding
+            
+            # CRITICAL: Use the correct model name
+            self.model_name = config.embedding_model
+            
+            # FastEmbed uses specific model names
+            if self.model_name == "sentence-transformers/all-MiniLM-L6-v2":
+                # This is the correct model we must use
+                fastembed_model = "sentence-transformers/all-MiniLM-L6-v2"
+            else:
+                fastembed_model = self.model_name
+            
+            logger.info(f"Initializing FastEmbed with model: {fastembed_model}")
+            
+            self.model = TextEmbedding(model_name=fastembed_model)
+            self.dimension = config.embedding_dimension
+            self._initialized = True
+            
+            logger.info(f"FastEmbed initialized successfully with dimension {self.dimension}")
+            
+        except ImportError as e:
+            error = EmbeddingError(
+                "FastEmbed not installed. Install with: pip install fastembed",
+                provider="FastEmbed"
+            )
+            self.handle_initialization_error(error)
+            raise error
+        except Exception as e:
+            error = EmbeddingError(
+                f"Failed to initialize FastEmbed: {str(e)}",
+                provider="FastEmbed"
+            )
+            self.handle_initialization_error(error)
+            raise error
+    
+    def embed(self, texts: List[str]) -> List[List[float]]:
+        """Generate embeddings for texts using FastEmbed."""
+        if not self._initialized:
+            raise EmbeddingError("FastEmbed not initialized", provider="FastEmbed")
+        
+        try:
+            # FastEmbed returns a generator, convert to list
+            embeddings = list(self.model.embed(texts))
+            
+            # Convert to regular Python lists with safe indexing
+            result = []
+            for i, embedding in enumerate(embeddings):
+                # Convert numpy array or similar to list
+                if hasattr(embedding, 'tolist'):
+                    emb_list = embedding.tolist()
+                else:
+                    emb_list = list(embedding)
+                
+                # Validate each embedding
+                if not self.validate_embedding(emb_list):
+                    # Safe indexing - use i which is guaranteed to be valid
+                    text_len = len(texts[i]) if i < len(texts) else 0
+                    raise EmbeddingError(
+                        f"Invalid embedding generated for text {i} of length {text_len}",
+                        provider="FastEmbed"
+                    )
+                
+                result.append(emb_list)
+            
+            return result
+            
+        except Exception as e:
+            if not isinstance(e, EmbeddingError):
+                e = EmbeddingError(
+                    f"Failed to generate embeddings: {str(e)}",
+                    provider="FastEmbed"
+                )
+            self._last_error = e
+            raise e
+    
+    def get_dimension(self) -> int:
+        """Get embedding dimension."""
+        if not self._initialized:
+            raise EmbeddingError("FastEmbed not initialized", provider="FastEmbed")
+        return self.dimension
+    
+    def validate_embedding(self, embedding: List[float]) -> bool:
+        """
+        Validate embedding quality.
+        
+        Checks:
+        1. Non-empty
+        2. Correct dimension
+        3. Not degenerate (all same values)
+        4. Has reasonable variance
+        """
+        if not embedding:
+            logger.error("Empty embedding detected")
+            return False
+        
+        # Check dimension
+        if len(embedding) != self.dimension:
+            logger.error(
+                f"Dimension mismatch: expected {self.dimension}, got {len(embedding)}"
+            )
+            return False
+        
+        # Check for degenerate embedding (all values identical)
+        unique_values = len(set(embedding))
+        if unique_values == 1:
+            logger.error(f"Degenerate embedding detected (all values are {embedding[0]})")
+            return False
+        
+        # Check variance is above threshold
+        try:
+            variance = statistics.variance(embedding)
+            if variance < 1e-6:
+                logger.warning(f"Low variance embedding detected: {variance}")
+                # Don't fail on low variance, just warn
+        except statistics.StatisticsError:
+            # Less than 2 data points
+            pass
+        
+        # Check for NaN or Inf values
+        if any(not isinstance(x, (int, float)) or x != x or abs(x) == float('inf') 
+               for x in embedding):
+            logger.error("Embedding contains NaN or Inf values")
+            return False
+        
+        return True
+    
+    def handle_initialization_error(self, error: Exception) -> None:
+        """Handle and log initialization errors."""
+        self._last_error = error
+        self._initialized = False
+        logger.error(f"FastEmbed initialization failed: {error}")
+        
+        # Could implement retry logic or fallback here
+        if "not installed" in str(error):
+            logger.info("Try: pip install fastembed")
+        elif "model" in str(error).lower():
+            logger.info(f"Model {self.model_name} may need to be downloaded first")

package/scripts/importer/embeddings/validator.py

@@ -0,0 +1,136 @@
+"""Embedding validation utilities."""
+
+import statistics
+from typing import List, Tuple, Optional
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class EmbeddingValidator:
+    """
+    Comprehensive embedding validation.
+    
+    Performs multiple checks to ensure embedding quality.
+    """
+    
+    def __init__(
+        self,
+        expected_dimension: int,
+        min_variance: float = 1e-6,
+        max_magnitude: float = 100.0
+    ):
+        self.expected_dimension = expected_dimension
+        self.min_variance = min_variance
+        self.max_magnitude = max_magnitude
+    
+    def validate(self, embedding: List[float]) -> Tuple[bool, Optional[str]]:
+        """
+        Validate an embedding vector.
+        
+        Returns:
+            Tuple of (is_valid, error_message)
+        """
+        # Check empty
+        if not embedding:
+            return False, "Empty embedding"
+        
+        # Check dimension
+        if len(embedding) != self.expected_dimension:
+            return False, f"Dimension mismatch: expected {self.expected_dimension}, got {len(embedding)}"
+        
+        # Check for NaN/Inf
+        for i, val in enumerate(embedding):
+            if not isinstance(val, (int, float)):
+                return False, f"Non-numeric value at index {i}: {type(val)}"
+            if val != val:  # NaN check
+                return False, f"NaN value at index {i}"
+            if abs(val) == float('inf'):
+                return False, f"Infinite value at index {i}"
+        
+        # Check for degenerate (all same)
+        unique_count = len(set(embedding))
+        if unique_count == 1:
+            return False, f"Degenerate embedding (all values are {embedding[0]})"
+        
+        # Check variance
+        if len(embedding) > 1:
+            try:
+                variance = statistics.variance(embedding)
+                if variance < self.min_variance:
+                    # Warning, not error
+                    logger.warning(f"Low variance: {variance}")
+            except Exception as e:
+                logger.warning(f"Could not calculate variance: {e}")
+        
+        # Check magnitude
+        max_val = max(abs(v) for v in embedding)
+        if max_val > self.max_magnitude:
+            return False, f"Value exceeds maximum magnitude: {max_val}"
+        
+        # Check for mostly zeros
+        zero_count = sum(1 for v in embedding if abs(v) < 1e-10)
+        if zero_count > len(embedding) * 0.9:
+            return False, f"Embedding is mostly zeros ({zero_count}/{len(embedding)})"
+        
+        return True, None
+    
+    def validate_batch(
+        self, 
+        embeddings: List[List[float]]
+    ) -> List[Tuple[int, str]]:
+        """
+        Validate a batch of embeddings.
+        
+        Returns:
+            List of (index, error_message) for invalid embeddings
+        """
+        errors = []
+        for i, embedding in enumerate(embeddings):
+            valid, error = self.validate(embedding)
+            if not valid:
+                errors.append((i, error))
+        return errors
+    
+    def check_similarity(
+        self,
+        embeddings: List[List[float]]
+    ) -> bool:
+        """
+        Check if embeddings in a batch are too similar.
+        
+        This can indicate a problem with the embedding model.
+        """
+        if len(embeddings) < 2:
+            return True
+        
+        # Calculate pairwise cosine similarities
+        from math import sqrt
+        
+        def cosine_similarity(a: List[float], b: List[float]) -> float:
+            dot_product = sum(x * y for x, y in zip(a, b))
+            norm_a = sqrt(sum(x * x for x in a))
+            norm_b = sqrt(sum(y * y for y in b))
+            if norm_a == 0 or norm_b == 0:
+                return 0
+            return dot_product / (norm_a * norm_b)
+        
+        # Check if all embeddings are too similar
+        high_similarity_count = 0
+        total_pairs = 0
+        
+        for i in range(len(embeddings)):
+            for j in range(i + 1, min(i + 5, len(embeddings))):  # Check first 5 pairs
+                similarity = cosine_similarity(embeddings[i], embeddings[j])
+                if similarity > 0.99:  # Nearly identical
+                    high_similarity_count += 1
+                total_pairs += 1
+        
+        if total_pairs > 0 and high_similarity_count / total_pairs > 0.8:
+            logger.warning(
+                f"High similarity detected: {high_similarity_count}/{total_pairs} "
+                f"pairs have >0.99 similarity"
+            )
+            return False
+        
+        return True

package/scripts/importer/embeddings/voyage_provider.py

@@ -0,0 +1,251 @@
+"""Voyage AI embedding provider with conditional import support."""
+
+import logging
+from typing import List, Optional
+
+from .base import EmbeddingProvider
+
+logger = logging.getLogger(__name__)
+
+# Conditional import to avoid dependency when not using Voyage
+try:
+    import voyageai
+    VOYAGE_AVAILABLE = True
+except ImportError:
+    voyageai = None
+    VOYAGE_AVAILABLE = False
+    logger.debug("Voyage AI not installed. Install with: pip install voyageai")
+
+
+class VoyageEmbeddingProvider(EmbeddingProvider):
+    """
+    Voyage AI cloud embedding provider.
+    
+    Supports multiple models with different dimensions:
+    - voyage-2: 1024 dimensions (default)
+    - voyage-large-2: 1536 dimensions
+    - voyage-3: 1024 dimensions
+    - voyage-3-lite: 512 dimensions
+    """
+    
+    SUPPORTED_MODELS = {
+        "voyage-2": 1024,
+        "voyage-large-2": 1536,
+        "voyage-3": 1024,
+        "voyage-3-lite": 512,
+    }
+    
+    def __init__(
+        self,
+        api_key: str,
+        model_name: str = "voyage-2",
+        batch_size: int = 128,
+        max_tokens_per_batch: int = 100000,  # 20k buffer from 120k limit
+        token_estimation_ratio: int = 3  # chars per token
+    ):
+        """
+        Initialize Voyage AI provider.
+        
+        Args:
+            api_key: Voyage AI API key
+            model_name: Model to use (default: voyage-2)
+            batch_size: Maximum batch size for embedding
+            max_tokens_per_batch: Maximum tokens per batch (default: 100000, 20k buffer from 120k limit)
+            token_estimation_ratio: Characters per token estimate (default: 3)
+            
+        Raises:
+            ImportError: If voyageai package is not installed
+            ValueError: If API key is empty or model is unsupported
+        """
+        if not VOYAGE_AVAILABLE:
+            raise ImportError(
+                "Voyage AI is not installed. "
+                "Install with: pip install voyageai"
+            )
+        
+        if not api_key:
+            raise ValueError("Voyage API key is required")
+        
+        if model_name not in self.SUPPORTED_MODELS:
+            raise ValueError(
+                f"Unsupported model: {model_name}. "
+                f"Supported models: {list(self.SUPPORTED_MODELS.keys())}"
+            )
+        
+        self.api_key = api_key
+        self.model_name = model_name
+        self.batch_size = batch_size
+        self.dimension = self.SUPPORTED_MODELS[model_name]
+        self.max_tokens_per_batch = max_tokens_per_batch
+        self.token_estimation_ratio = token_estimation_ratio
+        
+        # Initialize client
+        self.client = voyageai.Client(api_key=api_key)
+        logger.info(f"Initialized Voyage AI with model {model_name} ({self.dimension} dims)")
+        # Note: Never log the API key for security
+    
+    def embed(self, texts: List[str]) -> List[List[float]]:
+        """
+        Generate embeddings for texts using Voyage AI.
+        
+        Args:
+            texts: List of texts to embed
+            
+        Returns:
+            List of embedding vectors
+            
+        Raises:
+            ValueError: If request is invalid
+            Exception: If API call fails
+        """
+        if not texts:
+            return []
+        
+        try:
+            # Voyage AI expects a list of texts
+            result = self.client.embed(
+                texts=texts,
+                model=self.model_name
+            )
+            
+            # Extract embeddings from response
+            embeddings = result.embeddings
+            
+            # Validate dimensions
+            for i, embedding in enumerate(embeddings):
+                if len(embedding) != self.dimension:
+                    raise ValueError(
+                        f"Embedding {i} has wrong dimension: "
+                        f"expected {self.dimension}, got {len(embedding)}"
+                    )
+            
+            logger.debug(f"Generated {len(embeddings)} embeddings with Voyage AI")
+            return embeddings
+            
+        except AttributeError as e:
+            # Handle voyageai-specific errors if available
+            if 'RateLimitError' in str(type(e).__name__):
+                logger.error(f"Rate limit exceeded, retry with backoff")
+                raise
+            elif 'InvalidRequestError' in str(type(e).__name__):
+                logger.error(f"Invalid request to Voyage API")
+                raise ValueError(f"Invalid embedding request: {e}")
+            else:
+                logger.error(f"Voyage API error: {e}")
+                raise
+        except Exception as e:
+            logger.error(f"Unexpected error during embedding: {type(e).__name__}")
+            raise
+    
+    def estimate_tokens(self, text: str) -> int:
+        """
+        Estimate token count for text.
+        Conservative estimate: 3 characters = 1 token.
+        
+        Args:
+            text: Text to estimate tokens for
+            
+        Returns:
+            Estimated token count
+        """
+        return len(text) // self.token_estimation_ratio
+    
+    def embed_batch(self, texts: List[str]) -> List[List[float]]:
+        """
+        Generate embeddings in token-aware batches to respect API limits.
+        
+        This implements the critical fix for issue #38 - prevents
+        "max allowed tokens per batch is 120000" errors.
+        
+        Args:
+            texts: List of texts to embed
+            
+        Returns:
+            List of embedding vectors
+        """
+        if not texts:
+            return []
+        
+        all_embeddings = []
+        current_batch = []
+        current_tokens = 0
+        
+        for text in texts:
+            # Estimate tokens for this text
+            text_tokens = self.estimate_tokens(text)
+            
+            # Check if single text exceeds limit
+            if text_tokens > self.max_tokens_per_batch:
+                logger.warning(
+                    f"Single text with {text_tokens} estimated tokens exceeds "
+                    f"limit of {self.max_tokens_per_batch}. Truncating."
+                )
+                # Truncate text to fit within limit
+                max_chars = self.max_tokens_per_batch * self.token_estimation_ratio
+                text = text[:max_chars]
+                text_tokens = self.estimate_tokens(text)
+            
+            # Check if adding this text would exceed batch limit
+            if current_batch and (current_tokens + text_tokens) > self.max_tokens_per_batch:
+                # Process current batch
+                logger.debug(
+                    f"Processing batch with {len(current_batch)} texts, "
+                    f"~{current_tokens} tokens"
+                )
+                embeddings = self.embed(current_batch)
+                all_embeddings.extend(embeddings)
+                
+                # Start new batch
+                current_batch = [text]
+                current_tokens = text_tokens
+            else:
+                # Add to current batch
+                current_batch.append(text)
+                current_tokens += text_tokens
+        
+        # Process final batch
+        if current_batch:
+            logger.debug(
+                f"Processing final batch with {len(current_batch)} texts, "
+                f"~{current_tokens} tokens"
+            )
+            embeddings = self.embed(current_batch)
+            all_embeddings.extend(embeddings)
+        
+        return all_embeddings
+    
+    def get_dimension(self) -> int:
+        """Get embedding dimension for current model."""
+        return self.dimension
+    
+    def get_model_name(self) -> str:
+        """Get the model name being used."""
+        return self.model_name
+    
+    def validate_api_key(self) -> bool:
+        """
+        Validate that the API key works.
+        
+        Returns:
+            True if API key is valid
+        """
+        try:
+            # Test with a simple embedding
+            test_result = self.client.embed(
+                texts=["test"],
+                model=self.model_name
+            )
+            return len(test_result.embeddings) > 0
+        except Exception as e:
+            logger.error(f"API key validation failed: {e}")
+            return False
+    
+    @classmethod
+    def is_available(cls) -> bool:
+        """Check if Voyage AI is available for use."""
+        return VOYAGE_AVAILABLE
+    
+    @classmethod
+    def get_supported_models(cls) -> dict:
+        """Get dictionary of supported models and their dimensions."""
+        return cls.SUPPORTED_MODELS.copy()