claude-self-reflect 3.0.0 → 3.0.2

package/scripts/importer/core/models.py

@@ -0,0 +1,184 @@
+"""Core domain models for the import system."""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import List, Dict, Any, Optional, Set
+from uuid import UUID, uuid4
+
+
+@dataclass
+class Message:
+    """A single message in a conversation."""
+    
+    role: str
+    content: str
+    timestamp: Optional[datetime] = None
+    message_index: int = 0
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    
+    def __post_init__(self):
+        """Validate message on creation."""
+        if not self.role:
+            raise ValueError("Message role cannot be empty")
+        if self.role not in {"user", "assistant", "system", "human"}:
+            # Allow common variations
+            pass  # Log warning but don't fail
+        if self.message_index < 0:
+            raise ValueError(f"Message index cannot be negative: {self.message_index}")
+
+
+@dataclass
+class ConversationChunk:
+    """A chunk of conversation ready for embedding."""
+    
+    text: str
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    message_indices: List[int] = field(default_factory=list)
+    chunk_index: int = 0
+    total_chunks: int = 1
+    conversation_id: str = field(default_factory=lambda: str(uuid4()))
+    
+    def __post_init__(self):
+        """Validate chunk on creation."""
+        if not self.text:
+            raise ValueError("Chunk text cannot be empty")
+        if self.chunk_index < 0:
+            raise ValueError(f"Chunk index cannot be negative: {self.chunk_index}")
+        if self.chunk_index >= self.total_chunks:
+            raise ValueError(
+                f"Chunk index ({self.chunk_index}) must be less than "
+                f"total chunks ({self.total_chunks})"
+            )
+    
+    @property
+    def unique_id(self) -> str:
+        """Generate unique ID for this chunk."""
+        return f"{self.conversation_id}_{self.chunk_index}"
+    
+    def add_metadata(self, key: str, value: Any) -> None:
+        """Add metadata with conflict detection."""
+        if key in self.metadata:
+            # Handle conflict - could merge, replace, or raise
+            if isinstance(self.metadata[key], list) and isinstance(value, list):
+                # Merge lists
+                self.metadata[key] = list(set(self.metadata[key] + value))
+            else:
+                # Replace value (log warning in production)
+                self.metadata[key] = value
+        else:
+            self.metadata[key] = value
+
+
+@dataclass
+class ProcessedPoint:
+    """A fully processed point ready for storage."""
+    
+    id: str
+    vector: List[float]
+    payload: Dict[str, Any]
+    
+    def __post_init__(self):
+        """Validate point on creation."""
+        if not self.id:
+            raise ValueError("Point ID cannot be empty")
+        if not self.vector:
+            raise ValueError("Point vector cannot be empty")
+        if not isinstance(self.vector, list):
+            raise TypeError(f"Vector must be a list, got {type(self.vector)}")
+        if not all(isinstance(x, (int, float)) for x in self.vector):
+            raise TypeError("Vector must contain only numeric values")
+        if not self.payload:
+            # Empty payload is allowed but unusual
+            pass
+    
+    @property
+    def dimension(self) -> int:
+        """Get vector dimension."""
+        return len(self.vector)
+    
+    def validate_dimension(self, expected: int) -> bool:
+        """Check if vector has expected dimension."""
+        return self.dimension == expected
+
+
+@dataclass
+class ImportResult:
+    """Result of an import operation."""
+    
+    file_path: str
+    success: bool
+    points_created: int = 0
+    chunks_processed: int = 0
+    error: Optional[str] = None
+    duration_seconds: float = 0.0
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    
+    @property
+    def failed(self) -> bool:
+        """Check if import failed."""
+        return not self.success
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for serialization."""
+        return {
+            "file_path": self.file_path,
+            "success": self.success,
+            "points_created": self.points_created,
+            "chunks_processed": self.chunks_processed,
+            "error": self.error,
+            "duration_seconds": self.duration_seconds,
+            "metadata": self.metadata
+        }
+
+
+@dataclass
+class ImportStats:
+    """Aggregate statistics for import operations."""
+    
+    total_files: int = 0
+    successful_files: int = 0
+    failed_files: int = 0
+    total_points: int = 0
+    total_chunks: int = 0
+    total_duration_seconds: float = 0.0
+    errors: List[str] = field(default_factory=list)
+    
+    def add_result(self, result: ImportResult) -> None:
+        """Add a result to the statistics."""
+        self.total_files += 1
+        if result.success:
+            self.successful_files += 1
+            self.total_points += result.points_created
+            self.total_chunks += result.chunks_processed
+        else:
+            self.failed_files += 1
+            if result.error:
+                self.errors.append(f"{result.file_path}: {result.error}")
+        self.total_duration_seconds += result.duration_seconds
+    
+    @property
+    def success_rate(self) -> float:
+        """Calculate success rate."""
+        if self.total_files == 0:
+            return 0.0
+        return self.successful_files / self.total_files * 100
+    
+    @property
+    def average_duration(self) -> float:
+        """Calculate average import duration."""
+        if self.total_files == 0:
+            return 0.0
+        return self.total_duration_seconds / self.total_files
+    
+    def summary(self) -> str:
+        """Generate summary string."""
+        return (
+            f"Import Statistics:\n"
+            f"  Total Files: {self.total_files}\n"
+            f"  Successful: {self.successful_files} ({self.success_rate:.1f}%)\n"
+            f"  Failed: {self.failed_files}\n"
+            f"  Total Points: {self.total_points}\n"
+            f"  Total Chunks: {self.total_chunks}\n"
+            f"  Total Duration: {self.total_duration_seconds:.2f}s\n"
+            f"  Average Duration: {self.average_duration:.2f}s per file"
+        )

package/scripts/importer/embeddings/__init__.py

@@ -0,0 +1,22 @@
+"""Embedding providers for text vectorization."""
+
+from .base import EmbeddingProvider
+from .fastembed_provider import FastEmbedProvider
+from .validator import EmbeddingValidator
+
+# Conditional import for Voyage
+try:
+    from .voyage_provider import VoyageEmbeddingProvider
+    __all__ = [
+        "EmbeddingProvider",
+        "FastEmbedProvider",
+        "VoyageEmbeddingProvider",
+        "EmbeddingValidator"
+    ]
+except ImportError:
+    # Voyage not available, continue without it
+    __all__ = [
+        "EmbeddingProvider",
+        "FastEmbedProvider",
+        "EmbeddingValidator"
+    ]

package/scripts/importer/embeddings/base.py

@@ -0,0 +1,141 @@
+"""Abstract base class for embedding providers."""
+
+from abc import ABC, abstractmethod
+from typing import List, Optional, Dict, Any
+from ..core.exceptions import EmbeddingError
+
+
+class EmbeddingProvider(ABC):
+    """
+    Abstract interface for embedding providers.
+    
+    Defines the contract that all embedding providers must implement,
+    including error handling methods as recommended by code review.
+    """
+    
+    def __init__(self):
+        self._last_error: Optional[Exception] = None
+        self._initialized: bool = False
+    
+    @abstractmethod
+    def initialize(self, config: Any) -> None:
+        """
+        Initialize the embedding provider.
+        
+        Args:
+            config: Configuration object with provider-specific settings
+            
+        Raises:
+            EmbeddingError: If initialization fails
+        """
+        pass
+    
+    @abstractmethod
+    def embed_texts(self, texts: List[str]) -> List[List[float]]:
+        """
+        Generate embeddings for a list of texts.
+        
+        Args:
+            texts: List of text strings to embed
+            
+        Returns:
+            List of embedding vectors
+            
+        Raises:
+            EmbeddingError: If embedding generation fails
+        """
+        pass
+    
+    @abstractmethod
+    def get_dimension(self) -> int:
+        """
+        Get the dimension of embeddings produced by this provider.
+        
+        Returns:
+            Embedding dimension
+        """
+        pass
+    
+    @abstractmethod
+    def validate_embedding(self, embedding: List[float]) -> bool:
+        """
+        Validate that an embedding is well-formed and not degenerate.
+        
+        Args:
+            embedding: Embedding vector to validate
+            
+        Returns:
+            True if embedding is valid, False otherwise
+        """
+        pass
+    
+    @abstractmethod
+    def handle_initialization_error(self, error: Exception) -> None:
+        """
+        Handle initialization failures.
+        
+        Args:
+            error: The exception that occurred during initialization
+        """
+        pass
+    
+    def get_last_error(self) -> Optional[Exception]:
+        """
+        Retrieve the last error for diagnostics.
+        
+        Returns:
+            Last exception that occurred, or None
+        """
+        return self._last_error
+    
+    def is_initialized(self) -> bool:
+        """
+        Check if provider is initialized and ready.
+        
+        Returns:
+            True if initialized, False otherwise
+        """
+        return self._initialized
+    
+    def get_provider_info(self) -> Dict[str, Any]:
+        """
+        Get information about this embedding provider.
+        
+        Returns:
+            Dictionary with provider metadata
+        """
+        return {
+            "provider": self.__class__.__name__,
+            "initialized": self._initialized,
+            "dimension": self.get_dimension() if self._initialized else None,
+            "has_error": self._last_error is not None
+        }
+    
+    def batch_embed_texts(
+        self, 
+        texts: List[str], 
+        batch_size: int = 32
+    ) -> List[List[float]]:
+        """
+        Generate embeddings in batches for memory efficiency.
+        
+        Args:
+            texts: List of texts to embed
+            batch_size: Number of texts to process at once
+            
+        Returns:
+            List of embedding vectors
+            
+        Raises:
+            EmbeddingError: If embedding generation fails
+        """
+        if not self._initialized:
+            raise EmbeddingError("Provider not initialized", provider=self.__class__.__name__)
+        
+        embeddings = []
+        for i in range(0, len(texts), batch_size):
+            batch = texts[i:i + batch_size]
+            batch_embeddings = self.embed_texts(batch)
+            embeddings.extend(batch_embeddings)
+        
+        return embeddings

package/scripts/importer/embeddings/fastembed_provider.py

@@ -0,0 +1,164 @@
+"""FastEmbed provider for local embeddings."""
+
+from typing import List, Any
+import logging
+import statistics
+from .base import EmbeddingProvider
+from ..core.exceptions import EmbeddingError
+
+logger = logging.getLogger(__name__)
+
+
+class FastEmbedProvider(EmbeddingProvider):
+    """
+    FastEmbed provider for generating embeddings locally.
+    
+    Uses sentence-transformers/all-MiniLM-L6-v2 model by default.
+    """
+    
+    def __init__(self):
+        super().__init__()
+        self.model = None
+        self.model_name = None
+        self.dimension = None
+    
+    def initialize(self, config: Any) -> None:
+        """Initialize FastEmbed with the specified model."""
+        try:
+            from fastembed import TextEmbedding
+            
+            # CRITICAL: Use the correct model name
+            self.model_name = config.embedding_model
+            
+            # FastEmbed uses specific model names
+            if self.model_name == "sentence-transformers/all-MiniLM-L6-v2":
+                # This is the correct model we must use
+                fastembed_model = "sentence-transformers/all-MiniLM-L6-v2"
+            else:
+                fastembed_model = self.model_name
+            
+            logger.info(f"Initializing FastEmbed with model: {fastembed_model}")
+            
+            self.model = TextEmbedding(model_name=fastembed_model)
+            self.dimension = config.embedding_dimension
+            self._initialized = True
+            
+            logger.info(f"FastEmbed initialized successfully with dimension {self.dimension}")
+            
+        except ImportError as e:
+            error = EmbeddingError(
+                "FastEmbed not installed. Install with: pip install fastembed",
+                provider="FastEmbed"
+            )
+            self.handle_initialization_error(error)
+            raise error
+        except Exception as e:
+            error = EmbeddingError(
+                f"Failed to initialize FastEmbed: {str(e)}",
+                provider="FastEmbed"
+            )
+            self.handle_initialization_error(error)
+            raise error
+    
+    def embed(self, texts: List[str]) -> List[List[float]]:
+        """Generate embeddings for texts using FastEmbed."""
+        if not self._initialized:
+            raise EmbeddingError("FastEmbed not initialized", provider="FastEmbed")
+        
+        try:
+            # FastEmbed returns a generator, convert to list
+            embeddings = list(self.model.embed(texts))
+            
+            # Convert to regular Python lists with safe indexing
+            result = []
+            for i, embedding in enumerate(embeddings):
+                # Convert numpy array or similar to list
+                if hasattr(embedding, 'tolist'):
+                    emb_list = embedding.tolist()
+                else:
+                    emb_list = list(embedding)
+                
+                # Validate each embedding
+                if not self.validate_embedding(emb_list):
+                    # Safe indexing - use i which is guaranteed to be valid
+                    text_len = len(texts[i]) if i < len(texts) else 0
+                    raise EmbeddingError(
+                        f"Invalid embedding generated for text {i} of length {text_len}",
+                        provider="FastEmbed"
+                    )
+                
+                result.append(emb_list)
+            
+            return result
+            
+        except Exception as e:
+            if not isinstance(e, EmbeddingError):
+                e = EmbeddingError(
+                    f"Failed to generate embeddings: {str(e)}",
+                    provider="FastEmbed"
+                )
+            self._last_error = e
+            raise e
+    
+    def get_dimension(self) -> int:
+        """Get embedding dimension."""
+        if not self._initialized:
+            raise EmbeddingError("FastEmbed not initialized", provider="FastEmbed")
+        return self.dimension
+    
+    def validate_embedding(self, embedding: List[float]) -> bool:
+        """
+        Validate embedding quality.
+        
+        Checks:
+        1. Non-empty
+        2. Correct dimension
+        3. Not degenerate (all same values)
+        4. Has reasonable variance
+        """
+        if not embedding:
+            logger.error("Empty embedding detected")
+            return False
+        
+        # Check dimension
+        if len(embedding) != self.dimension:
+            logger.error(
+                f"Dimension mismatch: expected {self.dimension}, got {len(embedding)}"
+            )
+            return False
+        
+        # Check for degenerate embedding (all values identical)
+        unique_values = len(set(embedding))
+        if unique_values == 1:
+            logger.error(f"Degenerate embedding detected (all values are {embedding[0]})")
+            return False
+        
+        # Check variance is above threshold
+        try:
+            variance = statistics.variance(embedding)
+            if variance < 1e-6:
+                logger.warning(f"Low variance embedding detected: {variance}")
+                # Don't fail on low variance, just warn
+        except statistics.StatisticsError:
+            # Less than 2 data points
+            pass
+        
+        # Check for NaN or Inf values
+        if any(not isinstance(x, (int, float)) or x != x or abs(x) == float('inf') 
+               for x in embedding):
+            logger.error("Embedding contains NaN or Inf values")
+            return False
+        
+        return True
+    
+    def handle_initialization_error(self, error: Exception) -> None:
+        """Handle and log initialization errors."""
+        self._last_error = error
+        self._initialized = False
+        logger.error(f"FastEmbed initialization failed: {error}")
+        
+        # Could implement retry logic or fallback here
+        if "not installed" in str(error):
+            logger.info("Try: pip install fastembed")
+        elif "model" in str(error).lower():
+            logger.info(f"Model {self.model_name} may need to be downloaded first")

package/scripts/importer/embeddings/validator.py

@@ -0,0 +1,136 @@
+"""Embedding validation utilities."""
+
+import statistics
+from typing import List, Tuple, Optional
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class EmbeddingValidator:
+    """
+    Comprehensive embedding validation.
+    
+    Performs multiple checks to ensure embedding quality.
+    """
+    
+    def __init__(
+        self,
+        expected_dimension: int,
+        min_variance: float = 1e-6,
+        max_magnitude: float = 100.0
+    ):
+        self.expected_dimension = expected_dimension
+        self.min_variance = min_variance
+        self.max_magnitude = max_magnitude
+    
+    def validate(self, embedding: List[float]) -> Tuple[bool, Optional[str]]:
+        """
+        Validate an embedding vector.
+        
+        Returns:
+            Tuple of (is_valid, error_message)
+        """
+        # Check empty
+        if not embedding:
+            return False, "Empty embedding"
+        
+        # Check dimension
+        if len(embedding) != self.expected_dimension:
+            return False, f"Dimension mismatch: expected {self.expected_dimension}, got {len(embedding)}"
+        
+        # Check for NaN/Inf
+        for i, val in enumerate(embedding):
+            if not isinstance(val, (int, float)):
+                return False, f"Non-numeric value at index {i}: {type(val)}"
+            if val != val:  # NaN check
+                return False, f"NaN value at index {i}"
+            if abs(val) == float('inf'):
+                return False, f"Infinite value at index {i}"
+        
+        # Check for degenerate (all same)
+        unique_count = len(set(embedding))
+        if unique_count == 1:
+            return False, f"Degenerate embedding (all values are {embedding[0]})"
+        
+        # Check variance
+        if len(embedding) > 1:
+            try:
+                variance = statistics.variance(embedding)
+                if variance < self.min_variance:
+                    # Warning, not error
+                    logger.warning(f"Low variance: {variance}")
+            except Exception as e:
+                logger.warning(f"Could not calculate variance: {e}")
+        
+        # Check magnitude
+        max_val = max(abs(v) for v in embedding)
+        if max_val > self.max_magnitude:
+            return False, f"Value exceeds maximum magnitude: {max_val}"
+        
+        # Check for mostly zeros
+        zero_count = sum(1 for v in embedding if abs(v) < 1e-10)
+        if zero_count > len(embedding) * 0.9:
+            return False, f"Embedding is mostly zeros ({zero_count}/{len(embedding)})"
+        
+        return True, None
+    
+    def validate_batch(
+        self, 
+        embeddings: List[List[float]]
+    ) -> List[Tuple[int, str]]:
+        """
+        Validate a batch of embeddings.
+        
+        Returns:
+            List of (index, error_message) for invalid embeddings
+        """
+        errors = []
+        for i, embedding in enumerate(embeddings):
+            valid, error = self.validate(embedding)
+            if not valid:
+                errors.append((i, error))
+        return errors
+    
+    def check_similarity(
+        self,
+        embeddings: List[List[float]]
+    ) -> bool:
+        """
+        Check if embeddings in a batch are too similar.
+        
+        This can indicate a problem with the embedding model.
+        """
+        if len(embeddings) < 2:
+            return True
+        
+        # Calculate pairwise cosine similarities
+        from math import sqrt
+        
+        def cosine_similarity(a: List[float], b: List[float]) -> float:
+            dot_product = sum(x * y for x, y in zip(a, b))
+            norm_a = sqrt(sum(x * x for x in a))
+            norm_b = sqrt(sum(y * y for y in b))
+            if norm_a == 0 or norm_b == 0:
+                return 0
+            return dot_product / (norm_a * norm_b)
+        
+        # Check if all embeddings are too similar
+        high_similarity_count = 0
+        total_pairs = 0
+        
+        for i in range(len(embeddings)):
+            for j in range(i + 1, min(i + 5, len(embeddings))):  # Check first 5 pairs
+                similarity = cosine_similarity(embeddings[i], embeddings[j])
+                if similarity > 0.99:  # Nearly identical
+                    high_similarity_count += 1
+                total_pairs += 1
+        
+        if total_pairs > 0 and high_similarity_count / total_pairs > 0.8:
+            logger.warning(
+                f"High similarity detected: {high_similarity_count}/{total_pairs} "
+                f"pairs have >0.99 similarity"
+            )
+            return False
+        
+        return True