claude-self-reflect 3.2.4 → 3.3.1

package/mcp-server/src/connection_pool.py

@@ -0,0 +1,286 @@
+"""
+Connection pooling for Qdrant client to improve performance and resource management.
+"""
+
+import asyncio
+from typing import Optional, Any
+from contextlib import asynccontextmanager
+import logging
+from qdrant_client import AsyncQdrantClient
+
+logger = logging.getLogger(__name__)
+
+
+class QdrantConnectionPool:
+    """
+    A connection pool for Qdrant clients with configurable size and timeout.
+    """
+    
+    def __init__(
+        self, 
+        url: str, 
+        pool_size: int = 10,
+        max_overflow: int = 5,
+        timeout: float = 30.0,
+        retry_attempts: int = 3,
+        retry_delay: float = 1.0
+    ):
+        """
+        Initialize the connection pool.
+        
+        Args:
+            url: Qdrant server URL
+            pool_size: Base number of connections to maintain
+            max_overflow: Additional connections that can be created if pool is exhausted
+            timeout: Timeout for acquiring a connection from the pool
+            retry_attempts: Number of retry attempts for failed operations
+            retry_delay: Delay between retry attempts (with exponential backoff)
+        """
+        self.url = url
+        self.pool_size = pool_size
+        self.max_overflow = max_overflow
+        self.timeout = timeout
+        self.retry_attempts = retry_attempts
+        self.retry_delay = retry_delay
+        
+        # Connection pool
+        self._pool = asyncio.Queue(maxsize=pool_size)
+        self._overflow_connections = []
+        self._semaphore = asyncio.Semaphore(pool_size + max_overflow)
+        self._initialized = False
+        self._lock = asyncio.Lock()
+        
+        # Statistics
+        self.stats = {
+            'connections_created': 0,
+            'connections_reused': 0,
+            'connections_failed': 0,
+            'overflow_used': 0,
+            'timeouts': 0
+        }
+    
+    async def initialize(self):
+        """Initialize the connection pool with base connections."""
+        async with self._lock:
+            if self._initialized:
+                return
+            
+            # Create initial pool connections
+            for _ in range(self.pool_size):
+                try:
+                    client = AsyncQdrantClient(url=self.url)
+                    await self._pool.put(client)
+                    self.stats['connections_created'] += 1
+                except Exception as e:
+                    logger.error(f"Failed to create initial connection: {e}")
+                    self.stats['connections_failed'] += 1
+            
+            self._initialized = True
+            logger.info(f"Connection pool initialized with {self._pool.qsize()} connections")
+    
+    @asynccontextmanager
+    async def acquire(self):
+        """
+        Acquire a connection from the pool.
+        
+        Yields:
+            AsyncQdrantClient instance
+        """
+        if not self._initialized:
+            await self.initialize()
+        
+        client = None
+        acquired_from_overflow = False
+        
+        try:
+            # Try to get a connection with timeout
+            try:
+                client = await asyncio.wait_for(
+                    self._pool.get(),
+                    timeout=self.timeout
+                )
+                self.stats['connections_reused'] += 1
+            except asyncio.TimeoutError:
+                # Pool is exhausted, try overflow
+                self.stats['timeouts'] += 1
+                
+                if len(self._overflow_connections) < self.max_overflow:
+                    # Create overflow connection
+                    logger.debug("Creating overflow connection")
+                    client = AsyncQdrantClient(url=self.url)
+                    self._overflow_connections.append(client)
+                    acquired_from_overflow = True
+                    self.stats['overflow_used'] += 1
+                    self.stats['connections_created'] += 1
+                else:
+                    raise RuntimeError("Connection pool exhausted and max overflow reached")
+            
+            # Yield the client for use
+            yield client
+            
+        finally:
+            # Return connection to pool
+            if client is not None:
+                if acquired_from_overflow:
+                    # Remove from overflow list
+                    if client in self._overflow_connections:
+                        self._overflow_connections.remove(client)
+                else:
+                    # Return to pool
+                    try:
+                        await self._pool.put(client)
+                    except asyncio.QueueFull:
+                        # This shouldn't happen, but handle gracefully
+                        logger.warning("Connection pool is full, closing extra connection")
+                        # In production, we might want to close the client here
+    
+    async def execute_with_retry(self, func, *args, **kwargs):
+        """
+        Execute a function with retry logic and exponential backoff.
+        
+        Args:
+            func: Async function to execute
+            *args: Positional arguments for the function
+            **kwargs: Keyword arguments for the function
+        
+        Returns:
+            Result from the function
+        """
+        last_exception = None
+        delay = self.retry_delay
+        
+        for attempt in range(self.retry_attempts):
+            try:
+                async with self.acquire() as client:
+                    # Pass the client as the first argument
+                    return await func(client, *args, **kwargs)
+            except Exception as e:
+                last_exception = e
+                if attempt < self.retry_attempts - 1:
+                    logger.warning(f"Attempt {attempt + 1} failed: {e}. Retrying in {delay}s...")
+                    await asyncio.sleep(delay)
+                    delay *= 2  # Exponential backoff
+                else:
+                    logger.error(f"All {self.retry_attempts} attempts failed: {e}")
+        
+        raise last_exception
+    
+    async def close(self):
+        """Close all connections in the pool."""
+        async with self._lock:
+            # Close all pooled connections
+            while not self._pool.empty():
+                try:
+                    client = await self._pool.get()
+                    # AsyncQdrantClient doesn't have a close method, but we can del it
+                    del client
+                except Exception as e:
+                    logger.error(f"Error closing connection: {e}")
+            
+            # Close overflow connections
+            for client in self._overflow_connections:
+                try:
+                    del client
+                except Exception as e:
+                    logger.error(f"Error closing overflow connection: {e}")
+            
+            self._overflow_connections.clear()
+            self._initialized = False
+            logger.info("Connection pool closed")
+    
+    def get_stats(self) -> dict:
+        """Get pool statistics."""
+        return {
+            **self.stats,
+            'current_pool_size': self._pool.qsize() if self._initialized else 0,
+            'overflow_active': len(self._overflow_connections),
+            'initialized': self._initialized
+        }
+
+
+# Circuit breaker implementation for additional resilience
+class CircuitBreaker:
+    """
+    Circuit breaker pattern to prevent cascading failures.
+    """
+    
+    def __init__(
+        self,
+        failure_threshold: int = 5,
+        recovery_timeout: float = 60.0,
+        expected_exception: type = Exception
+    ):
+        """
+        Initialize circuit breaker.
+        
+        Args:
+            failure_threshold: Number of failures before opening circuit
+            recovery_timeout: Time to wait before attempting recovery
+            expected_exception: Exception type to catch
+        """
+        self.failure_threshold = failure_threshold
+        self.recovery_timeout = recovery_timeout
+        self.expected_exception = expected_exception
+        
+        self._failure_count = 0
+        self._last_failure_time = None
+        self._state = 'closed'  # closed, open, half_open
+        self._lock = asyncio.Lock()
+    
+    async def call(self, func, *args, **kwargs):
+        """
+        Call a function through the circuit breaker.
+        
+        Args:
+            func: Async function to call
+            *args: Positional arguments
+            **kwargs: Keyword arguments
+        
+        Returns:
+            Result from function
+        
+        Raises:
+            CircuitBreakerOpen: If circuit is open
+        """
+        async with self._lock:
+            # Check circuit state
+            if self._state == 'open':
+                # Check if we should try half-open
+                if self._last_failure_time:
+                    time_since_failure = asyncio.get_event_loop().time() - self._last_failure_time
+                    if time_since_failure > self.recovery_timeout:
+                        self._state = 'half_open'
+                        logger.info("Circuit breaker entering half-open state")
+                    else:
+                        raise CircuitBreakerOpen(f"Circuit breaker is open (failures: {self._failure_count})")
+        
+        try:
+            # Attempt the call
+            result = await func(*args, **kwargs)
+            
+            # Success - update state
+            async with self._lock:
+                if self._state == 'half_open':
+                    self._state = 'closed'
+                    logger.info("Circuit breaker closed after successful recovery")
+                self._failure_count = 0
+                self._last_failure_time = None
+            
+            return result
+            
+        except self.expected_exception as e:
+            # Failure - update state
+            async with self._lock:
+                self._failure_count += 1
+                self._last_failure_time = asyncio.get_event_loop().time()
+                
+                if self._failure_count >= self.failure_threshold:
+                    self._state = 'open'
+                    logger.error(f"Circuit breaker opened after {self._failure_count} failures")
+                
+                raise e
+
+
+class CircuitBreakerOpen(Exception):
+    """Exception raised when circuit breaker is open."""
+    pass

package/mcp-server/src/decay_manager.py

@@ -0,0 +1,106 @@
+"""Decay calculation manager for Claude Self-Reflect MCP server."""
+
+import math
+from datetime import datetime, timezone
+from typing import List, Tuple, Optional
+try:
+    from .config import (
+        USE_DECAY,
+        DECAY_SCALE_DAYS,
+        DECAY_WEIGHT,
+        USE_NATIVE_DECAY,
+        logger
+    )
+except ImportError:
+    # Fallback for direct execution
+    import os
+    import logging
+    USE_DECAY = os.getenv('USE_DECAY', 'false').lower() == 'true'
+    DECAY_SCALE_DAYS = float(os.getenv('DECAY_SCALE_DAYS', '90'))
+    DECAY_WEIGHT = float(os.getenv('DECAY_WEIGHT', '0.3'))
+    USE_NATIVE_DECAY = os.getenv('USE_NATIVE_DECAY', 'false').lower() == 'true'
+    logger = logging.getLogger(__name__)
+
+class DecayManager:
+    """Manages memory decay calculations for search results."""
+    
+    def __init__(self):
+        self.scale_ms = DECAY_SCALE_DAYS * 24 * 60 * 60 * 1000
+        self.weight = DECAY_WEIGHT
+        self.use_decay = USE_DECAY
+        self.use_native = USE_NATIVE_DECAY
+    
+    def calculate_decay_score(
+        self, 
+        base_score: float, 
+        timestamp: str
+    ) -> float:
+        """Calculate decayed score for a single result."""
+        if not self.use_decay:
+            return base_score
+        
+        try:
+            # Parse timestamp
+            if timestamp.endswith('Z'):
+                timestamp = timestamp.replace('Z', '+00:00')
+            
+            result_time = datetime.fromisoformat(timestamp)
+            if result_time.tzinfo is None:
+                result_time = result_time.replace(tzinfo=timezone.utc)
+            
+            # Calculate age
+            now = datetime.now(timezone.utc)
+            age_ms = (now - result_time).total_seconds() * 1000
+            
+            # Calculate decay factor using half-life formula
+            # decay = exp(-ln(2) * age / half_life)
+            decay_factor = math.exp(-0.693147 * age_ms / self.scale_ms)
+            
+            # Apply decay with weight
+            final_score = base_score * (1 - self.weight) + base_score * self.weight * decay_factor
+            
+            return final_score
+            
+        except Exception as e:
+            logger.error(f"Failed to calculate decay: {e}")
+            return base_score
+    
+    def apply_decay_to_results(
+        self, 
+        results: List[Tuple[float, str, dict]]
+    ) -> List[Tuple[float, str, dict]]:
+        """Apply decay to a list of results and re-sort."""
+        if not self.use_decay:
+            return results
+        
+        decayed_results = []
+        for score, id_str, payload in results:
+            timestamp = payload.get('timestamp', datetime.now().isoformat())
+            decayed_score = self.calculate_decay_score(score, timestamp)
+            decayed_results.append((decayed_score, id_str, payload))
+        
+        # Re-sort by decayed score
+        decayed_results.sort(key=lambda x: x[0], reverse=True)
+        
+        return decayed_results
+    
+    def get_native_decay_config(self) -> Optional[dict]:
+        """Get configuration for native Qdrant decay."""
+        if not self.use_native:
+            return None
+        
+        return {
+            'scale_seconds': self.scale_ms / 1000,
+            'weight': self.weight,
+            'midpoint': 0.5  # Half-life semantics
+        }
+    
+    def should_use_decay(self, explicit_setting: Optional[int] = None) -> bool:
+        """Determine if decay should be used for a query."""
+        if explicit_setting is not None:
+            if explicit_setting == 1:
+                return True
+            elif explicit_setting == 0:
+                return False
+        
+        return self.use_decay

package/mcp-server/src/embedding_manager.py

@@ -16,16 +16,16 @@ class EmbeddingManager:
     """Manages embedding models with proper cache and lock handling."""
     
     def __init__(self):
-        self.model = None
-        self.model_type = None  # 'local' or 'voyage'
+        self.local_model = None
         self.voyage_client = None
-        
+        self.model_type = None  # Default model type ('local' or 'voyage')
+
         # Configuration
         self.prefer_local = os.getenv('PREFER_LOCAL_EMBEDDINGS', 'true').lower() == 'true'
         self.voyage_key = os.getenv('VOYAGE_KEY') or os.getenv('VOYAGE_KEY-2')
         self.embedding_model = os.getenv('EMBEDDING_MODEL', 'sentence-transformers/all-MiniLM-L6-v2')
         self.download_timeout = int(os.getenv('FASTEMBED_DOWNLOAD_TIMEOUT', '30'))
-        
+
         # Set cache directory to our controlled location
         self.cache_dir = Path(__file__).parent.parent / '.fastembed-cache'
         
@@ -50,27 +50,35 @@ class EmbeddingManager:
                 logger.warning(f"Error cleaning locks: {e}")
         
     def initialize(self) -> bool:
-        """Initialize embedding model based on user preference."""
-        logger.info("Initializing embedding manager...")
-        
+        """Initialize BOTH embedding models to support mixed collections."""
+        logger.info("Initializing embedding manager for dual-mode support...")
+
         # Clean up any stale locks first
         self._clean_stale_locks()
-        
-        if self.prefer_local:
-            # User wants local - try local only, don't fallback to cloud
-            if self._try_initialize_local():
-                return True
-            logger.error("Local embeddings failed and user prefers local - not falling back to cloud")
-            return False
+
+        # Initialize both models for mixed collection support
+        local_success = self._try_initialize_local()
+        voyage_success = False
+
+        if self.voyage_key:
+            voyage_success = self._try_initialize_voyage()
+
+        # Set default model type based on preference and availability
+        if self.prefer_local and local_success:
+            self.model_type = 'local'
+            logger.info("Default model set to LOCAL embeddings")
+        elif voyage_success:
+            self.model_type = 'voyage'
+            logger.info("Default model set to VOYAGE embeddings")
+        elif local_success:
+            self.model_type = 'local'
+            logger.info("Default model set to LOCAL embeddings (fallback)")
         else:
-            # User prefers Voyage AI
-            if self.voyage_key and self._try_initialize_voyage():
-                return True
-            logger.warning("Voyage AI failed, trying local as fallback...")
-            if self._try_initialize_local():
-                return True
-            logger.error("Both Voyage AI and local embeddings failed")
+            logger.error("Failed to initialize any embedding model")
             return False
+
+        logger.info(f"Embedding models available - Local: {local_success}, Voyage: {voyage_success}")
+        return True
     
     def _try_initialize_local(self) -> bool:
         """Try to initialize local FastEmbed model with timeout and optimizations."""
@@ -119,11 +127,10 @@ class EmbeddingManager:
                     from fastembed import TextEmbedding
                     # Initialize with optimized settings
                     # Note: FastEmbed uses these environment variables internally
-                    self.model = TextEmbedding(
+                    self.local_model = TextEmbedding(
                         model_name=self.embedding_model,
                         threads=1  # Single thread per worker to prevent over-subscription
                     )
-                    self.model_type = 'local'
                     success = True
                     logger.info(f"Successfully initialized local model: {self.embedding_model} with single-thread mode")
                 except Exception as e:
@@ -177,39 +184,48 @@ class EmbeddingManager:
             logger.error(f"Failed to initialize Voyage AI: {e}")
             return False
     
-    def embed(self, texts: Union[str, List[str]], input_type: str = "document") -> Optional[List[List[float]]]:
-        """Generate embeddings using the active model."""
-        if not self.model and not self.voyage_client:
-            logger.error("No embedding model initialized")
+    def embed(self, texts: Union[str, List[str]], input_type: str = "document", force_type: str = None) -> Optional[List[List[float]]]:
+        """Generate embeddings using the specified or default model."""
+        # Determine which model to use
+        use_type = force_type if force_type else self.model_type
+        logger.debug(f"Embedding with: force_type={force_type}, self.model_type={self.model_type}, use_type={use_type}")
+
+        if use_type == 'local' and not self.local_model:
+            logger.error("Local model not initialized")
             return None
-        
+        elif use_type == 'voyage' and not self.voyage_client:
+            logger.error("Voyage client not initialized")
+            return None
+
         # Ensure texts is a list
         if isinstance(texts, str):
             texts = [texts]
-        
+
         try:
-            if self.model_type == 'local':
+            if use_type == 'local':
                 # FastEmbed returns a generator, convert to list
-                embeddings = list(self.model.embed(texts))
+                embeddings = list(self.local_model.embed(texts))
                 return [emb.tolist() for emb in embeddings]
-            
-            elif self.model_type == 'voyage':
+
+            elif use_type == 'voyage':
+                # Always use voyage-3 for consistency with collection dimensions (1024)
                 result = self.voyage_client.embed(
                     texts=texts,
-                    model="voyage-3-lite" if input_type == "query" else "voyage-3",
+                    model="voyage-3",
                     input_type=input_type
                 )
                 return result.embeddings
-            
+
         except Exception as e:
-            logger.error(f"Error generating embeddings: {e}")
+            logger.error(f"Error generating embeddings with {use_type}: {e}")
             return None
     
-    def get_vector_dimension(self) -> int:
-        """Get the dimension of embeddings."""
-        if self.model_type == 'local':
+    def get_vector_dimension(self, force_type: str = None) -> int:
+        """Get the dimension of embeddings for a specific type."""
+        use_type = force_type if force_type else self.model_type
+        if use_type == 'local':
             return 384  # all-MiniLM-L6-v2 dimension
-        elif self.model_type == 'voyage':
+        elif use_type == 'voyage':
             return 1024  # voyage-3 dimension
         return 0
     
@@ -222,6 +238,14 @@ class EmbeddingManager:
             'prefer_local': self.prefer_local,
             'has_voyage_key': bool(self.voyage_key)
         }
+    
+    async def generate_embedding(self, text: str, force_type: str = None) -> Optional[List[float]]:
+        """Generate embedding for a single text (async wrapper for compatibility)."""
+        # Use the force_type if specified, otherwise use default
+        result = self.embed(text, input_type="query", force_type=force_type)
+        if result and len(result) > 0:
+            return result[0]
+        return None
 
 
 # Global instance