claude-self-reflect 3.2.3 → 3.3.0

package/mcp-server/src/embeddings_old.py

@@ -0,0 +1,141 @@
+"""Embedding generation module for Claude Self-Reflect MCP server."""
+
+import os
+import voyageai
+from typing import Dict, List, Optional, Any
+from fastembed import TextEmbedding
+from config import (
+    VOYAGE_API_KEY, 
+    VOYAGE_MODEL, 
+    LOCAL_MODEL,
+    PREFER_LOCAL_EMBEDDINGS,
+    logger
+)
+
+class EmbeddingManager:
+    """Manages embedding generation for both local and Voyage AI models."""
+    
+    def __init__(self):
+        self.local_model = None
+        self.voyage_client = None
+        self.embedding_cache = {}
+        
+        # Initialize based on preference
+        if PREFER_LOCAL_EMBEDDINGS or not VOYAGE_API_KEY:
+            self._init_local_model()
+        
+        if VOYAGE_API_KEY:
+            self._init_voyage_client()
+    
+    def _init_local_model(self):
+        """Initialize local FastEmbed model."""
+        try:
+            self.local_model = TextEmbedding(
+                model_name=LOCAL_MODEL,
+                cache_dir=str(os.path.expanduser("~/.cache/fastembed"))
+            )
+            logger.info(f"Initialized local embedding model: {LOCAL_MODEL}")
+        except Exception as e:
+            logger.error(f"Failed to initialize local model: {e}")
+    
+    def _init_voyage_client(self):
+        """Initialize Voyage AI client."""
+        try:
+            self.voyage_client = voyageai.Client(api_key=VOYAGE_API_KEY)
+            logger.info("Initialized Voyage AI client")
+        except Exception as e:
+            logger.error(f"Failed to initialize Voyage client: {e}")
+    
+    async def generate_embedding(
+        self, 
+        text: str, 
+        embedding_type: Optional[str] = None
+    ) -> Optional[List[float]]:
+        """Generate embedding for text using specified or default model."""
+        
+        # Use cache if available
+        cache_key = f"{embedding_type or 'default'}:{text[:100]}"
+        if cache_key in self.embedding_cache:
+            return self.embedding_cache[cache_key]
+        
+        # Determine which model to use
+        use_local = True
+        if embedding_type:
+            use_local = 'local' in embedding_type
+        elif not PREFER_LOCAL_EMBEDDINGS and self.voyage_client:
+            use_local = False
+        
+        try:
+            if use_local and self.local_model:
+                # Generate local embedding
+                embeddings = list(self.local_model.embed([text]))
+                if embeddings:
+                    embedding = list(embeddings[0])
+                    self.embedding_cache[cache_key] = embedding
+                    return embedding
+            
+            elif self.voyage_client:
+                # Generate Voyage embedding
+                result = self.voyage_client.embed(
+                    [text], 
+                    model=VOYAGE_MODEL,
+                    input_type="document"
+                )
+                if result.embeddings:
+                    embedding = result.embeddings[0]
+                    self.embedding_cache[cache_key] = embedding
+                    return embedding
+            
+        except Exception as e:
+            logger.error(f"Failed to generate embedding: {e}")
+        
+        return None
+    
+    async def generate_embeddings_batch(
+        self, 
+        texts: List[str], 
+        embedding_type: Optional[str] = None
+    ) -> Dict[str, List[float]]:
+        """Generate embeddings for multiple texts efficiently."""
+        results = {}
+        
+        # Determine which model to use
+        use_local = True
+        if embedding_type:
+            use_local = 'local' in embedding_type
+        elif not PREFER_LOCAL_EMBEDDINGS and self.voyage_client:
+            use_local = False
+        
+        try:
+            if use_local and self.local_model:
+                # Batch process with local model
+                embeddings = list(self.local_model.embed(texts))
+                for text, embedding in zip(texts, embeddings):
+                    results[text] = list(embedding)
+            
+            elif self.voyage_client:
+                # Batch process with Voyage
+                result = self.voyage_client.embed(
+                    texts,
+                    model=VOYAGE_MODEL,
+                    input_type="document"
+                )
+                for text, embedding in zip(texts, result.embeddings):
+                    results[text] = embedding
+        
+        except Exception as e:
+            logger.error(f"Failed to generate batch embeddings: {e}")
+        
+        return results
+    
+    def get_embedding_dimension(self, embedding_type: str = "local") -> int:
+        """Get the dimension of embeddings for a given type."""
+        if "local" in embedding_type:
+            return 384  # all-MiniLM-L6-v2 dimension
+        else:
+            return 1024  # voyage-3-lite dimension
+    
+    def clear_cache(self):
+        """Clear the embedding cache."""
+        self.embedding_cache.clear()
+        logger.info("Cleared embedding cache")

package/mcp-server/src/models.py

@@ -0,0 +1,64 @@
+"""Pydantic models for Claude Self-Reflect MCP server."""
+
+from typing import Optional, List, Dict, Any, Set
+from datetime import datetime
+from pydantic import BaseModel, Field
+
+class SearchResult(BaseModel):
+    """Model for search results."""
+    id: str
+    score: float
+    timestamp: str
+    role: str
+    excerpt: str
+    project_name: str
+    conversation_id: Optional[str] = None
+    base_conversation_id: Optional[str] = None
+    collection_name: str
+    raw_payload: Optional[Dict[str, Any]] = None
+    code_patterns: Optional[Dict[str, List[str]]] = None
+    files_analyzed: Optional[List[str]] = None
+    files_edited: Optional[List[str]] = None
+    tools_used: Optional[List[str]] = None
+    concepts: Optional[List[str]] = None
+
+class ConversationGroup(BaseModel):
+    """Model for grouped conversations."""
+    conversation_id: str
+    base_conversation_id: str
+    timestamp: datetime
+    message_count: int
+    excerpts: List[str]
+    files: Set[str] = Field(default_factory=set)
+    tools: Set[str] = Field(default_factory=set)
+    concepts: Set[str] = Field(default_factory=set)
+
+class WorkSession(BaseModel):
+    """Model for work sessions."""
+    start_time: datetime
+    end_time: datetime
+    conversations: List[ConversationGroup]
+    total_messages: int
+    files_touched: Set[str] = Field(default_factory=set)
+    tools_used: Set[str] = Field(default_factory=set)
+    concepts: Set[str] = Field(default_factory=set)
+
+class ActivityStats(BaseModel):
+    """Model for activity statistics."""
+    total_conversations: int
+    total_messages: int
+    unique_files: int
+    unique_tools: int
+    peak_hour: Optional[str] = None
+    peak_day: Optional[str] = None
+
+class TimelineEntry(BaseModel):
+    """Model for timeline entries."""
+    period: str
+    start_time: datetime
+    end_time: datetime
+    conversation_count: int
+    message_count: int
+    files: Set[str] = Field(default_factory=set)
+    tools: Set[str] = Field(default_factory=set)
+    concepts: Set[str] = Field(default_factory=set)

package/mcp-server/src/parallel_search.py

@@ -0,0 +1,371 @@
+"""
+Parallel search implementation for Qdrant collections.
+This module implements asyncio.gather-based parallel searching to improve performance.
+"""
+
+import asyncio
+import time
+from typing import List, Dict, Any, Optional, Tuple
+from datetime import datetime
+import logging
+
+logger = logging.getLogger(__name__)
+
+async def search_single_collection(
+    collection_name: str,
+    query: str,
+    query_embeddings: Dict[str, List[float]],
+    qdrant_client: Any,
+    ctx: Any,
+    limit: int,
+    min_score: float,
+    should_use_decay: bool,
+    target_project: str,
+    generate_embedding_func: Any,
+    constants: Dict[str, Any]
+) -> Tuple[str, List[Any], Dict[str, Any]]:
+    """
+    Search a single collection and return results.
+    
+    Returns:
+        Tuple of (collection_name, results, timing_info)
+    """
+    collection_timing = {'name': collection_name, 'start': time.time()}
+    results = []
+    
+    try:
+        # Determine embedding type for this collection
+        embedding_type_for_collection = 'voyage' if collection_name.endswith('_voyage') else 'local'
+        logger.debug(f"Collection {collection_name} needs {embedding_type_for_collection} embedding")
+
+        # Generate or retrieve cached embedding for this type
+        if embedding_type_for_collection not in query_embeddings:
+            try:
+                query_embeddings[embedding_type_for_collection] = await generate_embedding_func(
+                    query, force_type=embedding_type_for_collection
+                )
+            except Exception as e:
+                await ctx.debug(f"Failed to generate {embedding_type_for_collection} embedding: {e}")
+                collection_timing['error'] = str(e)
+                collection_timing['end'] = time.time()
+                return collection_name, results, collection_timing
+        
+        query_embedding = query_embeddings[embedding_type_for_collection]
+        
+        # Check if this is a reflections collection
+        is_reflection_collection = collection_name.startswith('reflections_')
+        
+        # Import necessary models
+        from qdrant_client import models
+        
+        # Determine which decay method to use
+        USE_NATIVE_DECAY = constants.get('USE_NATIVE_DECAY', False)
+        NATIVE_DECAY_AVAILABLE = constants.get('NATIVE_DECAY_AVAILABLE', False)
+        DECAY_SCALE_DAYS = constants.get('DECAY_SCALE_DAYS', 90)
+        DECAY_WEIGHT = constants.get('DECAY_WEIGHT', 0.3)
+        
+        if should_use_decay and USE_NATIVE_DECAY and NATIVE_DECAY_AVAILABLE:
+            # Use native Qdrant decay with newer API
+            await ctx.debug(f"Using NATIVE Qdrant decay (new API) for {collection_name}")
+            
+            half_life_seconds = DECAY_SCALE_DAYS * 24 * 60 * 60
+            
+            # Build query using Qdrant's Fusion and RankFusion
+            fusion_query = models.Fusion(
+                fusion=models.RankFusion.RRF,
+                queries=[
+                    # Semantic similarity query
+                    models.NearestQuery(
+                        nearest=query_embedding,
+                        score_threshold=min_score
+                    ),
+                    # Time decay query using context pair
+                    models.ContextQuery(
+                        context=[
+                            models.ContextPair(
+                                positive=models.DiscoverQuery(
+                                    target=query_embedding,
+                                    context=[
+                                        models.ContextPair(
+                                            positive=models.DatetimeRange(
+                                                gt=datetime.now().isoformat(),
+                                                lt=(datetime.now().timestamp() + half_life_seconds)
+                                            )
+                                        )
+                                    ]
+                                )
+                            )
+                        ]
+                    )
+                ]
+            )
+            
+            # Execute search with native decay
+            search_results = await qdrant_client.query_points(
+                collection_name=collection_name,
+                query=fusion_query,
+                limit=limit,
+                with_payload=True
+            )
+            
+            # Process results
+            for point in search_results.points:
+                # Process each point and add to results
+                raw_timestamp = point.payload.get('timestamp', datetime.now().isoformat())
+                clean_timestamp = raw_timestamp.replace('Z', '+00:00') if raw_timestamp.endswith('Z') else raw_timestamp
+                
+                point_project = point.payload.get('project', collection_name.replace('conv_', '').replace('_voyage', '').replace('_local', ''))
+                
+                # Apply project filtering
+                if target_project != 'all' and not is_reflection_collection:
+                    if point_project != target_project:
+                        normalized_target = target_project.replace('-', '_')
+                        normalized_point = point_project.replace('-', '_')
+                        if not (normalized_point == normalized_target or 
+                                point_project.endswith(f"/{target_project}") or
+                                point_project.endswith(f"-{target_project}") or
+                                normalized_point.endswith(f"_{normalized_target}") or
+                                normalized_point.endswith(f"/{normalized_target}")):
+                            continue
+                
+                # Create SearchResult
+                search_result = {
+                    'id': str(point.id),
+                    'score': point.score,
+                    'timestamp': clean_timestamp,
+                    'role': point.payload.get('start_role', point.payload.get('role', 'unknown')),
+                    'excerpt': (point.payload.get('text', '')[:350] + '...' 
+                               if len(point.payload.get('text', '')) > 350 
+                               else point.payload.get('text', '')),
+                    'project_name': point_project,
+                    'payload': point.payload
+                }
+                results.append(search_result)
+                
+        else:
+            # Standard search without native decay or client-side decay
+            search_results = await qdrant_client.search(
+                collection_name=collection_name,
+                query_vector=query_embedding,
+                limit=limit * 3 if should_use_decay else limit,  # Get more results for client-side decay
+                score_threshold=min_score if not should_use_decay else 0.0,
+                with_payload=True
+            )
+
+            # Debug: Log search results
+            logger.debug(f"Search of {collection_name} returned {len(search_results)} results")
+            
+            if should_use_decay and not USE_NATIVE_DECAY:
+                # Apply client-side decay
+                await ctx.debug(f"Using CLIENT-SIDE decay for {collection_name}")
+                decay_results = []
+                
+                for point in search_results:
+                    try:
+                        raw_timestamp = point.payload.get('timestamp', datetime.now().isoformat())
+                        clean_timestamp = raw_timestamp.replace('Z', '+00:00') if raw_timestamp.endswith('Z') else raw_timestamp
+                        
+                        # Calculate age and decay
+                        if 'timestamp' in point.payload:
+                            try:
+                                point_time = datetime.fromisoformat(clean_timestamp)
+                                if point_time.tzinfo is None:
+                                    from zoneinfo import ZoneInfo
+                                    point_time = point_time.replace(tzinfo=ZoneInfo('UTC'))
+                                
+                                now = datetime.now(ZoneInfo('UTC'))
+                                age_ms = (now - point_time).total_seconds() * 1000
+                                
+                                # Exponential decay with configurable half-life
+                                half_life_ms = DECAY_SCALE_DAYS * 24 * 60 * 60 * 1000
+                                decay_factor = 0.5 ** (age_ms / half_life_ms)
+                                
+                                # Apply multiplicative decay
+                                adjusted_score = point.score * ((1 - DECAY_WEIGHT) + DECAY_WEIGHT * decay_factor)
+                            except Exception as e:
+                                await ctx.debug(f"Error calculating decay: {e}")
+                                adjusted_score = point.score
+                        else:
+                            adjusted_score = point.score
+                        
+                        # Only include if above min_score after decay
+                        if adjusted_score >= min_score:
+                            decay_results.append((adjusted_score, point))
+                    
+                    except Exception as e:
+                        await ctx.debug(f"Error applying decay to point: {e}")
+                        decay_results.append((point.score, point))
+                
+                # Sort by adjusted score and take top results
+                decay_results.sort(key=lambda x: x[0], reverse=True)
+                
+                # Convert to SearchResult format
+                for adjusted_score, point in decay_results[:limit]:
+                    raw_timestamp = point.payload.get('timestamp', datetime.now().isoformat())
+                    clean_timestamp = raw_timestamp.replace('Z', '+00:00') if raw_timestamp.endswith('Z') else raw_timestamp
+                    
+                    point_project = point.payload.get('project', collection_name.replace('conv_', '').replace('_voyage', '').replace('_local', ''))
+                    
+                    # Apply project filtering
+                    if target_project != 'all' and not is_reflection_collection:
+                        if point_project != target_project:
+                            normalized_target = target_project.replace('-', '_')
+                            normalized_point = point_project.replace('-', '_')
+                            if not (normalized_point == normalized_target or
+                                    point_project.endswith(f"/{target_project}") or
+                                    point_project.endswith(f"-{target_project}") or
+                                    normalized_point.endswith(f"_{normalized_target}") or
+                                    normalized_point.endswith(f"/{normalized_target}")):
+                                logger.debug(f"Filtering out point: project '{point_project}' != target '{target_project}'")
+                                continue
+                        logger.debug(f"Keeping point: project '{point_project}' matches target '{target_project}'")
+                    
+                    # Create SearchResult
+                    search_result = {
+                        'id': str(point.id),
+                        'score': adjusted_score,
+                        'timestamp': clean_timestamp,
+                        'role': point.payload.get('start_role', point.payload.get('role', 'unknown')),
+                        'excerpt': (point.payload.get('text', '')[:350] + '...' 
+                                   if len(point.payload.get('text', '')) > 350 
+                                   else point.payload.get('text', '')),
+                        'project_name': point_project,
+                        'payload': point.payload
+                    }
+                    results.append(search_result)
+            else:
+                # Process standard search results without decay
+                logger.debug(f"Processing {len(search_results)} results from {collection_name}")
+                for point in search_results:
+                    raw_timestamp = point.payload.get('timestamp', datetime.now().isoformat())
+                    clean_timestamp = raw_timestamp.replace('Z', '+00:00') if raw_timestamp.endswith('Z') else raw_timestamp
+
+                    point_project = point.payload.get('project', collection_name.replace('conv_', '').replace('_voyage', '').replace('_local', ''))
+                    
+                    # Apply project filtering
+                    if target_project != 'all' and not is_reflection_collection:
+                        if point_project != target_project:
+                            normalized_target = target_project.replace('-', '_')
+                            normalized_point = point_project.replace('-', '_')
+                            if not (normalized_point == normalized_target or
+                                    point_project.endswith(f"/{target_project}") or
+                                    point_project.endswith(f"-{target_project}") or
+                                    normalized_point.endswith(f"_{normalized_target}") or
+                                    normalized_point.endswith(f"/{normalized_target}")):
+                                logger.debug(f"Filtering out point: project '{point_project}' != target '{target_project}'")
+                                continue
+                        logger.debug(f"Keeping point: project '{point_project}' matches target '{target_project}'")
+                    
+                    # Create SearchResult as dictionary (consistent with other branches)
+                    search_result = {
+                        'id': str(point.id),
+                        'score': point.score,
+                        'timestamp': clean_timestamp,
+                        'role': point.payload.get('start_role', point.payload.get('role', 'unknown')),
+                        'excerpt': (point.payload.get('text', '')[:350] + '...'
+                                   if len(point.payload.get('text', '')) > 350
+                                   else point.payload.get('text', '')),
+                        'project_name': point_project,
+                        'conversation_id': point.payload.get('conversation_id'),
+                        'base_conversation_id': point.payload.get('base_conversation_id'),
+                        'collection_name': collection_name,
+                        'raw_payload': point.payload,
+                        'code_patterns': point.payload.get('code_patterns'),
+                        'files_analyzed': point.payload.get('files_analyzed'),
+                        'tools_used': list(point.payload.get('tools_used', [])) if isinstance(point.payload.get('tools_used'), set) else point.payload.get('tools_used'),
+                        'concepts': point.payload.get('concepts')
+                    }
+                    results.append(search_result)
+    
+    except Exception as e:
+        await ctx.debug(f"Error searching {collection_name}: {str(e)}")
+        collection_timing['error'] = str(e)
+    
+    collection_timing['end'] = time.time()
+    logger.debug(f"Collection {collection_name} returning {len(results)} results after filtering")
+    return collection_name, results, collection_timing
+
+
+async def parallel_search_collections(
+    collections_to_search: List[str],
+    query: str,
+    qdrant_client: Any,
+    ctx: Any,
+    limit: int,
+    min_score: float,
+    should_use_decay: bool,
+    target_project: str,
+    generate_embedding_func: Any,
+    constants: Dict[str, Any],
+    max_concurrent: int = 10
+) -> Tuple[List[Any], List[Dict[str, Any]]]:
+    """
+    Search multiple collections in parallel using asyncio.gather.
+    
+    Args:
+        collections_to_search: List of collection names to search
+        query: Search query
+        qdrant_client: Qdrant client instance
+        ctx: Context for debugging
+        limit: Maximum results per collection
+        min_score: Minimum similarity score
+        should_use_decay: Whether to apply time decay
+        target_project: Project filter ('all' or specific project)
+        generate_embedding_func: Function to generate embeddings
+        constants: Dictionary of constants (USE_NATIVE_DECAY, etc.)
+        max_concurrent: Maximum concurrent searches
+    
+    Returns:
+        Tuple of (all_results, collection_timings)
+    """
+    await ctx.debug(f"Starting parallel search across {len(collections_to_search)} collections")
+    
+    # Shared cache for embeddings
+    query_embeddings = {}
+    
+    # Create semaphore to limit concurrent operations
+    semaphore = asyncio.Semaphore(max_concurrent)
+    
+    async def search_with_semaphore(collection_name: str) -> Tuple[str, List[Any], Dict[str, Any]]:
+        """Search with concurrency limit"""
+        async with semaphore:
+            return await search_single_collection(
+                collection_name=collection_name,
+                query=query,
+                query_embeddings=query_embeddings,
+                qdrant_client=qdrant_client,
+                ctx=ctx,
+                limit=limit,
+                min_score=min_score,
+                should_use_decay=should_use_decay,
+                target_project=target_project,
+                generate_embedding_func=generate_embedding_func,
+                constants=constants
+            )
+    
+    # Launch all searches in parallel
+    search_tasks = [
+        search_with_semaphore(collection_name)
+        for collection_name in collections_to_search
+    ]
+    
+    # Wait for all searches to complete
+    search_results = await asyncio.gather(*search_tasks, return_exceptions=True)
+    
+    # Process results
+    all_results = []
+    collection_timings = []
+    
+    for result in search_results:
+        if isinstance(result, Exception):
+            # Handle exceptions from gather
+            logger.error(f"Search task failed: {result}")
+            continue
+        
+        collection_name, results, timing = result
+        all_results.extend(results)
+        collection_timings.append(timing)
+    
+    await ctx.debug(f"Parallel search complete: {len(all_results)} total results")
+    
+    return all_results, collection_timings