claude-self-reflect 3.0.0 → 3.0.1

package/mcp-server/pyproject.toml

@@ -15,6 +15,7 @@ dependencies = [
     "pydantic>=2.11.7,<3.0.0",  # Updated for fastmcp 2.10.6 compatibility
     "pydantic-settings>=2.0.0,<3.0.0",
     "fastembed>=0.4.0,<1.0.0",
+    "dependency-injector>=4.0.0,<5.0.0",
 ]
 
 [project.scripts]

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-self-reflect",
-  "version": "3.0.0",
+  "version": "3.0.1",
   "description": "Give Claude perfect memory of all your conversations - Installation wizard for Python MCP server",
   "keywords": [
     "claude",
@@ -41,6 +41,7 @@
     "mcp-server/run-mcp-clean.sh",
     "mcp-server/run-mcp-docker.sh",
     "scripts/import-*.py",
+    "scripts/importer/**/*.py",
     "scripts/delta-metadata-update-safe.py",
     "scripts/force-metadata-recovery.py",
     ".claude/agents/*.md",

package/scripts/importer/__init__.py

@@ -0,0 +1,25 @@
+"""
+Claude Self-Reflect Modular Import System
+==========================================
+
+A pristine, modular conversation import system following SOLID principles
+and clean architecture patterns.
+
+Version: 3.0.0
+Author: Claude Self-Reflect Team
+License: MIT
+"""
+
+from .core.config import ImportConfig
+from .core.models import Message, ConversationChunk, ProcessedPoint
+from .main import ConversationProcessor, ImporterContainer
+
+__version__ = "3.0.0"
+__all__ = [
+    "ImportConfig",
+    "Message", 
+    "ConversationChunk",
+    "ProcessedPoint",
+    "ConversationProcessor",
+    "ImporterContainer"
+]

package/scripts/importer/__main__.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+"""Entry point for running the importer as a module."""
+
+import sys
+import logging
+from pathlib import Path
+
+# Add parent directory to path for standalone execution
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from importer.main import main
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/importer/core/__init__.py

@@ -0,0 +1,25 @@
+"""Core domain models and configuration."""
+
+from .config import ImportConfig
+from .models import Message, ConversationChunk, ProcessedPoint, ImportResult, ImportStats
+from .exceptions import (
+    ImportError,
+    ValidationError, 
+    EmbeddingError,
+    StorageError,
+    ParseError
+)
+
+__all__ = [
+    "ImportConfig",
+    "Message",
+    "ConversationChunk", 
+    "ProcessedPoint",
+    "ImportResult",
+    "ImportStats",
+    "ImportError",
+    "ValidationError",
+    "EmbeddingError",
+    "StorageError",
+    "ParseError"
+]

package/scripts/importer/core/config.py

@@ -0,0 +1,120 @@
+"""Immutable configuration with validation."""
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Optional
+import os
+
+
+@dataclass(frozen=True)
+class ImportConfig:
+    """
+    Immutable configuration for the import system.
+    
+    All validation happens in __post_init__ to ensure configuration
+    is always in a valid state.
+    """
+    
+    # Qdrant settings
+    qdrant_url: str = field(default="http://localhost:6333")
+    qdrant_api_key: Optional[str] = field(default=None)
+    
+    # Embedding settings
+    embedding_model: str = field(default="sentence-transformers/all-MiniLM-L6-v2")
+    embedding_dimension: int = field(default=384)
+    use_voyage: bool = field(default=False)
+    voyage_api_key: Optional[str] = field(default=None)
+    
+    # Chunking settings
+    chunk_size: int = field(default=3000)
+    chunk_overlap: int = field(default=200)
+    
+    # Processing settings
+    batch_size: int = field(default=10)
+    max_ast_elements: int = field(default=100)
+    max_workers: int = field(default=4)
+    
+    # State management
+    state_file: str = field(default="~/.claude-self-reflect/config/imported-files.json")
+    
+    # Operational settings
+    log_level: str = field(default="INFO")
+    dry_run: bool = field(default=False)
+    force_reimport: bool = field(default=False)
+    
+    # Limits
+    file_limit: Optional[int] = field(default=None)
+    
+    def __post_init__(self):
+        """Validate configuration on initialization."""
+        # Validate chunk settings
+        if self.chunk_size <= 0:
+            raise ValueError(f"chunk_size must be positive, got {self.chunk_size}")
+        
+        if self.chunk_overlap < 0:
+            raise ValueError(f"chunk_overlap cannot be negative, got {self.chunk_overlap}")
+        
+        if self.chunk_overlap >= self.chunk_size:
+            raise ValueError(
+                f"chunk_overlap ({self.chunk_overlap}) must be less than "
+                f"chunk_size ({self.chunk_size})"
+            )
+        
+        # Validate batch settings
+        if self.batch_size < 1:
+            raise ValueError(f"batch_size must be at least 1, got {self.batch_size}")
+        
+        if self.max_workers < 1:
+            raise ValueError(f"max_workers must be at least 1, got {self.max_workers}")
+        
+        # Validate embedding settings
+        if self.embedding_dimension <= 0:
+            raise ValueError(f"embedding_dimension must be positive, got {self.embedding_dimension}")
+        
+        if self.use_voyage and not self.voyage_api_key:
+            # Document the limitation of frozen dataclass
+            voyage_key = os.getenv("VOYAGE_KEY")
+            if not voyage_key:
+                raise ValueError(
+                    "voyage_api_key must be provided at initialization when use_voyage=True. "
+                    "Set VOYAGE_KEY environment variable before creating config."
+                )
+        
+        # Validate log level
+        valid_levels = {"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"}
+        if self.log_level.upper() not in valid_levels:
+            raise ValueError(f"log_level must be one of {valid_levels}, got {self.log_level}")
+    
+    @property
+    def state_file_path(self) -> Path:
+        """Get expanded state file path with fallback."""
+        try:
+            return Path(self.state_file).expanduser()
+        except (RuntimeError, OSError):
+            # Fallback to current directory if expansion fails
+            return Path.cwd() / ".import-state.json"
+    
+    @classmethod
+    def from_env(cls) -> "ImportConfig":
+        """Create configuration from environment variables."""
+        return cls(
+            qdrant_url=os.getenv("QDRANT_URL", "http://localhost:6333"),
+            qdrant_api_key=os.getenv("QDRANT_API_KEY"),
+            use_voyage=os.getenv("USE_VOYAGE", "false").lower() == "true",
+            voyage_api_key=os.getenv("VOYAGE_KEY"),
+            chunk_size=int(os.getenv("CHUNK_SIZE", "3000")),
+            chunk_overlap=int(os.getenv("CHUNK_OVERLAP", "200")),
+            batch_size=int(os.getenv("BATCH_SIZE", "10")),
+            max_workers=int(os.getenv("MAX_WORKERS", "4")),
+            log_level=os.getenv("LOG_LEVEL", "INFO"),
+            dry_run=os.getenv("DRY_RUN", "false").lower() == "true",
+            force_reimport=os.getenv("FORCE_REIMPORT", "false").lower() == "true"
+        )
+    
+    @classmethod
+    def from_dict(cls, config_dict: dict) -> "ImportConfig":
+        """Create configuration from dictionary."""
+        # Filter out any unknown keys
+        known_fields = {f.name for f in cls.__dataclass_fields__.values()}
+        filtered_dict = {k: v for k, v in config_dict.items() if k in known_fields}
+        return cls(**filtered_dict)

package/scripts/importer/core/exceptions.py

@@ -0,0 +1,52 @@
+"""Custom exception hierarchy for import system."""
+
+from typing import Optional, Any
+
+
+class ImportError(Exception):
+    """Base exception for all import-related errors."""
+    
+    def __init__(self, message: str, details: Optional[dict] = None):
+        super().__init__(message)
+        self.details = details or {}
+
+
+class ValidationError(ImportError):
+    """Raised when input validation fails."""
+    
+    def __init__(self, field: str, value: Any, reason: str):
+        super().__init__(f"Validation failed for {field}: {reason}")
+        self.field = field
+        self.value = value
+        self.reason = reason
+
+
+class EmbeddingError(ImportError):
+    """Raised when embedding generation or validation fails."""
+    
+    def __init__(self, message: str, provider: Optional[str] = None):
+        super().__init__(message)
+        self.provider = provider
+
+
+class StorageError(ImportError):
+    """Raised when storage operations fail."""
+    
+    def __init__(self, operation: str, collection: str, reason: str):
+        super().__init__(f"Storage {operation} failed for {collection}: {reason}")
+        self.operation = operation
+        self.collection = collection
+
+
+class ParseError(ImportError):
+    """Raised when parsing conversation files fails."""
+    
+    def __init__(self, file_path: str, line_number: Optional[int] = None, reason: str = ""):
+        message = f"Failed to parse {file_path}"
+        if line_number:
+            message += f" at line {line_number}"
+        if reason:
+            message += f": {reason}"
+        super().__init__(message)
+        self.file_path = file_path
+        self.line_number = line_number

package/scripts/importer/core/models.py

@@ -0,0 +1,184 @@
+"""Core domain models for the import system."""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import List, Dict, Any, Optional, Set
+from uuid import UUID, uuid4
+
+
+@dataclass
+class Message:
+    """A single message in a conversation."""
+    
+    role: str
+    content: str
+    timestamp: Optional[datetime] = None
+    message_index: int = 0
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    
+    def __post_init__(self):
+        """Validate message on creation."""
+        if not self.role:
+            raise ValueError("Message role cannot be empty")
+        if self.role not in {"user", "assistant", "system", "human"}:
+            # Allow common variations
+            pass  # Log warning but don't fail
+        if self.message_index < 0:
+            raise ValueError(f"Message index cannot be negative: {self.message_index}")
+
+
+@dataclass
+class ConversationChunk:
+    """A chunk of conversation ready for embedding."""
+    
+    text: str
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    message_indices: List[int] = field(default_factory=list)
+    chunk_index: int = 0
+    total_chunks: int = 1
+    conversation_id: str = field(default_factory=lambda: str(uuid4()))
+    
+    def __post_init__(self):
+        """Validate chunk on creation."""
+        if not self.text:
+            raise ValueError("Chunk text cannot be empty")
+        if self.chunk_index < 0:
+            raise ValueError(f"Chunk index cannot be negative: {self.chunk_index}")
+        if self.chunk_index >= self.total_chunks:
+            raise ValueError(
+                f"Chunk index ({self.chunk_index}) must be less than "
+                f"total chunks ({self.total_chunks})"
+            )
+    
+    @property
+    def unique_id(self) -> str:
+        """Generate unique ID for this chunk."""
+        return f"{self.conversation_id}_{self.chunk_index}"
+    
+    def add_metadata(self, key: str, value: Any) -> None:
+        """Add metadata with conflict detection."""
+        if key in self.metadata:
+            # Handle conflict - could merge, replace, or raise
+            if isinstance(self.metadata[key], list) and isinstance(value, list):
+                # Merge lists
+                self.metadata[key] = list(set(self.metadata[key] + value))
+            else:
+                # Replace value (log warning in production)
+                self.metadata[key] = value
+        else:
+            self.metadata[key] = value
+
+
+@dataclass
+class ProcessedPoint:
+    """A fully processed point ready for storage."""
+    
+    id: str
+    vector: List[float]
+    payload: Dict[str, Any]
+    
+    def __post_init__(self):
+        """Validate point on creation."""
+        if not self.id:
+            raise ValueError("Point ID cannot be empty")
+        if not self.vector:
+            raise ValueError("Point vector cannot be empty")
+        if not isinstance(self.vector, list):
+            raise TypeError(f"Vector must be a list, got {type(self.vector)}")
+        if not all(isinstance(x, (int, float)) for x in self.vector):
+            raise TypeError("Vector must contain only numeric values")
+        if not self.payload:
+            # Empty payload is allowed but unusual
+            pass
+    
+    @property
+    def dimension(self) -> int:
+        """Get vector dimension."""
+        return len(self.vector)
+    
+    def validate_dimension(self, expected: int) -> bool:
+        """Check if vector has expected dimension."""
+        return self.dimension == expected
+
+
+@dataclass
+class ImportResult:
+    """Result of an import operation."""
+    
+    file_path: str
+    success: bool
+    points_created: int = 0
+    chunks_processed: int = 0
+    error: Optional[str] = None
+    duration_seconds: float = 0.0
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    
+    @property
+    def failed(self) -> bool:
+        """Check if import failed."""
+        return not self.success
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for serialization."""
+        return {
+            "file_path": self.file_path,
+            "success": self.success,
+            "points_created": self.points_created,
+            "chunks_processed": self.chunks_processed,
+            "error": self.error,
+            "duration_seconds": self.duration_seconds,
+            "metadata": self.metadata
+        }
+
+
+@dataclass
+class ImportStats:
+    """Aggregate statistics for import operations."""
+    
+    total_files: int = 0
+    successful_files: int = 0
+    failed_files: int = 0
+    total_points: int = 0
+    total_chunks: int = 0
+    total_duration_seconds: float = 0.0
+    errors: List[str] = field(default_factory=list)
+    
+    def add_result(self, result: ImportResult) -> None:
+        """Add a result to the statistics."""
+        self.total_files += 1
+        if result.success:
+            self.successful_files += 1
+            self.total_points += result.points_created
+            self.total_chunks += result.chunks_processed
+        else:
+            self.failed_files += 1
+            if result.error:
+                self.errors.append(f"{result.file_path}: {result.error}")
+        self.total_duration_seconds += result.duration_seconds
+    
+    @property
+    def success_rate(self) -> float:
+        """Calculate success rate."""
+        if self.total_files == 0:
+            return 0.0
+        return self.successful_files / self.total_files * 100
+    
+    @property
+    def average_duration(self) -> float:
+        """Calculate average import duration."""
+        if self.total_files == 0:
+            return 0.0
+        return self.total_duration_seconds / self.total_files
+    
+    def summary(self) -> str:
+        """Generate summary string."""
+        return (
+            f"Import Statistics:\n"
+            f"  Total Files: {self.total_files}\n"
+            f"  Successful: {self.successful_files} ({self.success_rate:.1f}%)\n"
+            f"  Failed: {self.failed_files}\n"
+            f"  Total Points: {self.total_points}\n"
+            f"  Total Chunks: {self.total_chunks}\n"
+            f"  Total Duration: {self.total_duration_seconds:.2f}s\n"
+            f"  Average Duration: {self.average_duration:.2f}s per file"
+        )

package/scripts/importer/embeddings/__init__.py

@@ -0,0 +1,22 @@
+"""Embedding providers for text vectorization."""
+
+from .base import EmbeddingProvider
+from .fastembed_provider import FastEmbedProvider
+from .validator import EmbeddingValidator
+
+# Conditional import for Voyage
+try:
+    from .voyage_provider import VoyageEmbeddingProvider
+    __all__ = [
+        "EmbeddingProvider",
+        "FastEmbedProvider",
+        "VoyageEmbeddingProvider",
+        "EmbeddingValidator"
+    ]
+except ImportError:
+    # Voyage not available, continue without it
+    __all__ = [
+        "EmbeddingProvider",
+        "FastEmbedProvider",
+        "EmbeddingValidator"
+    ]

package/scripts/importer/embeddings/base.py

@@ -0,0 +1,141 @@
+"""Abstract base class for embedding providers."""
+
+from abc import ABC, abstractmethod
+from typing import List, Optional, Dict, Any
+from ..core.exceptions import EmbeddingError
+
+
+class EmbeddingProvider(ABC):
+    """
+    Abstract interface for embedding providers.
+    
+    Defines the contract that all embedding providers must implement,
+    including error handling methods as recommended by code review.
+    """
+    
+    def __init__(self):
+        self._last_error: Optional[Exception] = None
+        self._initialized: bool = False
+    
+    @abstractmethod
+    def initialize(self, config: Any) -> None:
+        """
+        Initialize the embedding provider.
+        
+        Args:
+            config: Configuration object with provider-specific settings
+            
+        Raises:
+            EmbeddingError: If initialization fails
+        """
+        pass
+    
+    @abstractmethod
+    def embed_texts(self, texts: List[str]) -> List[List[float]]:
+        """
+        Generate embeddings for a list of texts.
+        
+        Args:
+            texts: List of text strings to embed
+            
+        Returns:
+            List of embedding vectors
+            
+        Raises:
+            EmbeddingError: If embedding generation fails
+        """
+        pass
+    
+    @abstractmethod
+    def get_dimension(self) -> int:
+        """
+        Get the dimension of embeddings produced by this provider.
+        
+        Returns:
+            Embedding dimension
+        """
+        pass
+    
+    @abstractmethod
+    def validate_embedding(self, embedding: List[float]) -> bool:
+        """
+        Validate that an embedding is well-formed and not degenerate.
+        
+        Args:
+            embedding: Embedding vector to validate
+            
+        Returns:
+            True if embedding is valid, False otherwise
+        """
+        pass
+    
+    @abstractmethod
+    def handle_initialization_error(self, error: Exception) -> None:
+        """
+        Handle initialization failures.
+        
+        Args:
+            error: The exception that occurred during initialization
+        """
+        pass
+    
+    def get_last_error(self) -> Optional[Exception]:
+        """
+        Retrieve the last error for diagnostics.
+        
+        Returns:
+            Last exception that occurred, or None
+        """
+        return self._last_error
+    
+    def is_initialized(self) -> bool:
+        """
+        Check if provider is initialized and ready.
+        
+        Returns:
+            True if initialized, False otherwise
+        """
+        return self._initialized
+    
+    def get_provider_info(self) -> Dict[str, Any]:
+        """
+        Get information about this embedding provider.
+        
+        Returns:
+            Dictionary with provider metadata
+        """
+        return {
+            "provider": self.__class__.__name__,
+            "initialized": self._initialized,
+            "dimension": self.get_dimension() if self._initialized else None,
+            "has_error": self._last_error is not None
+        }
+    
+    def batch_embed_texts(
+        self, 
+        texts: List[str], 
+        batch_size: int = 32
+    ) -> List[List[float]]:
+        """
+        Generate embeddings in batches for memory efficiency.
+        
+        Args:
+            texts: List of texts to embed
+            batch_size: Number of texts to process at once
+            
+        Returns:
+            List of embedding vectors
+            
+        Raises:
+            EmbeddingError: If embedding generation fails
+        """
+        if not self._initialized:
+            raise EmbeddingError("Provider not initialized", provider=self.__class__.__name__)
+        
+        embeddings = []
+        for i in range(0, len(texts), batch_size):
+            batch = texts[i:i + batch_size]
+            batch_embeddings = self.embed_texts(batch)
+            embeddings.extend(batch_embeddings)
+        
+        return embeddings