claude-self-reflect 3.2.4 → 3.3.0

package/mcp-server/src/parallel_search.py

@@ -0,0 +1,371 @@
+"""
+Parallel search implementation for Qdrant collections.
+This module implements asyncio.gather-based parallel searching to improve performance.
+"""
+
+import asyncio
+import time
+from typing import List, Dict, Any, Optional, Tuple
+from datetime import datetime
+import logging
+
+logger = logging.getLogger(__name__)
+
+async def search_single_collection(
+    collection_name: str,
+    query: str,
+    query_embeddings: Dict[str, List[float]],
+    qdrant_client: Any,
+    ctx: Any,
+    limit: int,
+    min_score: float,
+    should_use_decay: bool,
+    target_project: str,
+    generate_embedding_func: Any,
+    constants: Dict[str, Any]
+) -> Tuple[str, List[Any], Dict[str, Any]]:
+    """
+    Search a single collection and return results.
+    
+    Returns:
+        Tuple of (collection_name, results, timing_info)
+    """
+    collection_timing = {'name': collection_name, 'start': time.time()}
+    results = []
+    
+    try:
+        # Determine embedding type for this collection
+        embedding_type_for_collection = 'voyage' if collection_name.endswith('_voyage') else 'local'
+        logger.debug(f"Collection {collection_name} needs {embedding_type_for_collection} embedding")
+
+        # Generate or retrieve cached embedding for this type
+        if embedding_type_for_collection not in query_embeddings:
+            try:
+                query_embeddings[embedding_type_for_collection] = await generate_embedding_func(
+                    query, force_type=embedding_type_for_collection
+                )
+            except Exception as e:
+                await ctx.debug(f"Failed to generate {embedding_type_for_collection} embedding: {e}")
+                collection_timing['error'] = str(e)
+                collection_timing['end'] = time.time()
+                return collection_name, results, collection_timing
+        
+        query_embedding = query_embeddings[embedding_type_for_collection]
+        
+        # Check if this is a reflections collection
+        is_reflection_collection = collection_name.startswith('reflections_')
+        
+        # Import necessary models
+        from qdrant_client import models
+        
+        # Determine which decay method to use
+        USE_NATIVE_DECAY = constants.get('USE_NATIVE_DECAY', False)
+        NATIVE_DECAY_AVAILABLE = constants.get('NATIVE_DECAY_AVAILABLE', False)
+        DECAY_SCALE_DAYS = constants.get('DECAY_SCALE_DAYS', 90)
+        DECAY_WEIGHT = constants.get('DECAY_WEIGHT', 0.3)
+        
+        if should_use_decay and USE_NATIVE_DECAY and NATIVE_DECAY_AVAILABLE:
+            # Use native Qdrant decay with newer API
+            await ctx.debug(f"Using NATIVE Qdrant decay (new API) for {collection_name}")
+            
+            half_life_seconds = DECAY_SCALE_DAYS * 24 * 60 * 60
+            
+            # Build query using Qdrant's Fusion and RankFusion
+            fusion_query = models.Fusion(
+                fusion=models.RankFusion.RRF,
+                queries=[
+                    # Semantic similarity query
+                    models.NearestQuery(
+                        nearest=query_embedding,
+                        score_threshold=min_score
+                    ),
+                    # Time decay query using context pair
+                    models.ContextQuery(
+                        context=[
+                            models.ContextPair(
+                                positive=models.DiscoverQuery(
+                                    target=query_embedding,
+                                    context=[
+                                        models.ContextPair(
+                                            positive=models.DatetimeRange(
+                                                gt=datetime.now().isoformat(),
+                                                lt=(datetime.now().timestamp() + half_life_seconds)
+                                            )
+                                        )
+                                    ]
+                                )
+                            )
+                        ]
+                    )
+                ]
+            )
+            
+            # Execute search with native decay
+            search_results = await qdrant_client.query_points(
+                collection_name=collection_name,
+                query=fusion_query,
+                limit=limit,
+                with_payload=True
+            )
+            
+            # Process results
+            for point in search_results.points:
+                # Process each point and add to results
+                raw_timestamp = point.payload.get('timestamp', datetime.now().isoformat())
+                clean_timestamp = raw_timestamp.replace('Z', '+00:00') if raw_timestamp.endswith('Z') else raw_timestamp
+                
+                point_project = point.payload.get('project', collection_name.replace('conv_', '').replace('_voyage', '').replace('_local', ''))
+                
+                # Apply project filtering
+                if target_project != 'all' and not is_reflection_collection:
+                    if point_project != target_project:
+                        normalized_target = target_project.replace('-', '_')
+                        normalized_point = point_project.replace('-', '_')
+                        if not (normalized_point == normalized_target or 
+                                point_project.endswith(f"/{target_project}") or
+                                point_project.endswith(f"-{target_project}") or
+                                normalized_point.endswith(f"_{normalized_target}") or
+                                normalized_point.endswith(f"/{normalized_target}")):
+                            continue
+                
+                # Create SearchResult
+                search_result = {
+                    'id': str(point.id),
+                    'score': point.score,
+                    'timestamp': clean_timestamp,
+                    'role': point.payload.get('start_role', point.payload.get('role', 'unknown')),
+                    'excerpt': (point.payload.get('text', '')[:350] + '...' 
+                               if len(point.payload.get('text', '')) > 350 
+                               else point.payload.get('text', '')),
+                    'project_name': point_project,
+                    'payload': point.payload
+                }
+                results.append(search_result)
+                
+        else:
+            # Standard search without native decay or client-side decay
+            search_results = await qdrant_client.search(
+                collection_name=collection_name,
+                query_vector=query_embedding,
+                limit=limit * 3 if should_use_decay else limit,  # Get more results for client-side decay
+                score_threshold=min_score if not should_use_decay else 0.0,
+                with_payload=True
+            )
+
+            # Debug: Log search results
+            logger.debug(f"Search of {collection_name} returned {len(search_results)} results")
+            
+            if should_use_decay and not USE_NATIVE_DECAY:
+                # Apply client-side decay
+                await ctx.debug(f"Using CLIENT-SIDE decay for {collection_name}")
+                decay_results = []
+                
+                for point in search_results:
+                    try:
+                        raw_timestamp = point.payload.get('timestamp', datetime.now().isoformat())
+                        clean_timestamp = raw_timestamp.replace('Z', '+00:00') if raw_timestamp.endswith('Z') else raw_timestamp
+                        
+                        # Calculate age and decay
+                        if 'timestamp' in point.payload:
+                            try:
+                                point_time = datetime.fromisoformat(clean_timestamp)
+                                if point_time.tzinfo is None:
+                                    from zoneinfo import ZoneInfo
+                                    point_time = point_time.replace(tzinfo=ZoneInfo('UTC'))
+                                
+                                now = datetime.now(ZoneInfo('UTC'))
+                                age_ms = (now - point_time).total_seconds() * 1000
+                                
+                                # Exponential decay with configurable half-life
+                                half_life_ms = DECAY_SCALE_DAYS * 24 * 60 * 60 * 1000
+                                decay_factor = 0.5 ** (age_ms / half_life_ms)
+                                
+                                # Apply multiplicative decay
+                                adjusted_score = point.score * ((1 - DECAY_WEIGHT) + DECAY_WEIGHT * decay_factor)
+                            except Exception as e:
+                                await ctx.debug(f"Error calculating decay: {e}")
+                                adjusted_score = point.score
+                        else:
+                            adjusted_score = point.score
+                        
+                        # Only include if above min_score after decay
+                        if adjusted_score >= min_score:
+                            decay_results.append((adjusted_score, point))
+                    
+                    except Exception as e:
+                        await ctx.debug(f"Error applying decay to point: {e}")
+                        decay_results.append((point.score, point))
+                
+                # Sort by adjusted score and take top results
+                decay_results.sort(key=lambda x: x[0], reverse=True)
+                
+                # Convert to SearchResult format
+                for adjusted_score, point in decay_results[:limit]:
+                    raw_timestamp = point.payload.get('timestamp', datetime.now().isoformat())
+                    clean_timestamp = raw_timestamp.replace('Z', '+00:00') if raw_timestamp.endswith('Z') else raw_timestamp
+                    
+                    point_project = point.payload.get('project', collection_name.replace('conv_', '').replace('_voyage', '').replace('_local', ''))
+                    
+                    # Apply project filtering
+                    if target_project != 'all' and not is_reflection_collection:
+                        if point_project != target_project:
+                            normalized_target = target_project.replace('-', '_')
+                            normalized_point = point_project.replace('-', '_')
+                            if not (normalized_point == normalized_target or
+                                    point_project.endswith(f"/{target_project}") or
+                                    point_project.endswith(f"-{target_project}") or
+                                    normalized_point.endswith(f"_{normalized_target}") or
+                                    normalized_point.endswith(f"/{normalized_target}")):
+                                logger.debug(f"Filtering out point: project '{point_project}' != target '{target_project}'")
+                                continue
+                        logger.debug(f"Keeping point: project '{point_project}' matches target '{target_project}'")
+                    
+                    # Create SearchResult
+                    search_result = {
+                        'id': str(point.id),
+                        'score': adjusted_score,
+                        'timestamp': clean_timestamp,
+                        'role': point.payload.get('start_role', point.payload.get('role', 'unknown')),
+                        'excerpt': (point.payload.get('text', '')[:350] + '...' 
+                                   if len(point.payload.get('text', '')) > 350 
+                                   else point.payload.get('text', '')),
+                        'project_name': point_project,
+                        'payload': point.payload
+                    }
+                    results.append(search_result)
+            else:
+                # Process standard search results without decay
+                logger.debug(f"Processing {len(search_results)} results from {collection_name}")
+                for point in search_results:
+                    raw_timestamp = point.payload.get('timestamp', datetime.now().isoformat())
+                    clean_timestamp = raw_timestamp.replace('Z', '+00:00') if raw_timestamp.endswith('Z') else raw_timestamp
+
+                    point_project = point.payload.get('project', collection_name.replace('conv_', '').replace('_voyage', '').replace('_local', ''))
+                    
+                    # Apply project filtering
+                    if target_project != 'all' and not is_reflection_collection:
+                        if point_project != target_project:
+                            normalized_target = target_project.replace('-', '_')
+                            normalized_point = point_project.replace('-', '_')
+                            if not (normalized_point == normalized_target or
+                                    point_project.endswith(f"/{target_project}") or
+                                    point_project.endswith(f"-{target_project}") or
+                                    normalized_point.endswith(f"_{normalized_target}") or
+                                    normalized_point.endswith(f"/{normalized_target}")):
+                                logger.debug(f"Filtering out point: project '{point_project}' != target '{target_project}'")
+                                continue
+                        logger.debug(f"Keeping point: project '{point_project}' matches target '{target_project}'")
+                    
+                    # Create SearchResult as dictionary (consistent with other branches)
+                    search_result = {
+                        'id': str(point.id),
+                        'score': point.score,
+                        'timestamp': clean_timestamp,
+                        'role': point.payload.get('start_role', point.payload.get('role', 'unknown')),
+                        'excerpt': (point.payload.get('text', '')[:350] + '...'
+                                   if len(point.payload.get('text', '')) > 350
+                                   else point.payload.get('text', '')),
+                        'project_name': point_project,
+                        'conversation_id': point.payload.get('conversation_id'),
+                        'base_conversation_id': point.payload.get('base_conversation_id'),
+                        'collection_name': collection_name,
+                        'raw_payload': point.payload,
+                        'code_patterns': point.payload.get('code_patterns'),
+                        'files_analyzed': point.payload.get('files_analyzed'),
+                        'tools_used': list(point.payload.get('tools_used', [])) if isinstance(point.payload.get('tools_used'), set) else point.payload.get('tools_used'),
+                        'concepts': point.payload.get('concepts')
+                    }
+                    results.append(search_result)
+    
+    except Exception as e:
+        await ctx.debug(f"Error searching {collection_name}: {str(e)}")
+        collection_timing['error'] = str(e)
+    
+    collection_timing['end'] = time.time()
+    logger.debug(f"Collection {collection_name} returning {len(results)} results after filtering")
+    return collection_name, results, collection_timing
+
+
+async def parallel_search_collections(
+    collections_to_search: List[str],
+    query: str,
+    qdrant_client: Any,
+    ctx: Any,
+    limit: int,
+    min_score: float,
+    should_use_decay: bool,
+    target_project: str,
+    generate_embedding_func: Any,
+    constants: Dict[str, Any],
+    max_concurrent: int = 10
+) -> Tuple[List[Any], List[Dict[str, Any]]]:
+    """
+    Search multiple collections in parallel using asyncio.gather.
+    
+    Args:
+        collections_to_search: List of collection names to search
+        query: Search query
+        qdrant_client: Qdrant client instance
+        ctx: Context for debugging
+        limit: Maximum results per collection
+        min_score: Minimum similarity score
+        should_use_decay: Whether to apply time decay
+        target_project: Project filter ('all' or specific project)
+        generate_embedding_func: Function to generate embeddings
+        constants: Dictionary of constants (USE_NATIVE_DECAY, etc.)
+        max_concurrent: Maximum concurrent searches
+    
+    Returns:
+        Tuple of (all_results, collection_timings)
+    """
+    await ctx.debug(f"Starting parallel search across {len(collections_to_search)} collections")
+    
+    # Shared cache for embeddings
+    query_embeddings = {}
+    
+    # Create semaphore to limit concurrent operations
+    semaphore = asyncio.Semaphore(max_concurrent)
+    
+    async def search_with_semaphore(collection_name: str) -> Tuple[str, List[Any], Dict[str, Any]]:
+        """Search with concurrency limit"""
+        async with semaphore:
+            return await search_single_collection(
+                collection_name=collection_name,
+                query=query,
+                query_embeddings=query_embeddings,
+                qdrant_client=qdrant_client,
+                ctx=ctx,
+                limit=limit,
+                min_score=min_score,
+                should_use_decay=should_use_decay,
+                target_project=target_project,
+                generate_embedding_func=generate_embedding_func,
+                constants=constants
+            )
+    
+    # Launch all searches in parallel
+    search_tasks = [
+        search_with_semaphore(collection_name)
+        for collection_name in collections_to_search
+    ]
+    
+    # Wait for all searches to complete
+    search_results = await asyncio.gather(*search_tasks, return_exceptions=True)
+    
+    # Process results
+    all_results = []
+    collection_timings = []
+    
+    for result in search_results:
+        if isinstance(result, Exception):
+            # Handle exceptions from gather
+            logger.error(f"Search task failed: {result}")
+            continue
+        
+        collection_name, results, timing = result
+        all_results.extend(results)
+        collection_timings.append(timing)
+    
+    await ctx.debug(f"Parallel search complete: {len(all_results)} total results")
+    
+    return all_results, collection_timings

package/mcp-server/src/project_resolver.py

@@ -69,6 +69,11 @@ class ProjectResolver:
         Returns:
             List of collection names that match the project
         """
+        # Special case: 'all' returns all conversation collections
+        if user_project_name == 'all':
+            collection_names = self._get_collection_names()
+            return collection_names  # Return all conv_ collections
+        
         if user_project_name in self._cache:
             # Check if cache entry is still valid
             if time() - self._cache_ttl.get(user_project_name, 0) < self._cache_duration:

package/mcp-server/src/reflection_tools.py

@@ -0,0 +1,206 @@
+"""Reflection tools for Claude Self Reflect MCP server."""
+
+import os
+import json
+import hashlib
+import logging
+from typing import Optional, List, Dict, Any
+from datetime import datetime, timezone
+from pathlib import Path
+import uuid
+
+from fastmcp import Context
+from pydantic import Field
+from qdrant_client import AsyncQdrantClient
+from qdrant_client.models import PointStruct, VectorParams, Distance
+
+logger = logging.getLogger(__name__)
+
+
+class ReflectionTools:
+    """Handles reflection storage and conversation retrieval operations."""
+    
+    def __init__(
+        self,
+        qdrant_client: AsyncQdrantClient,
+        qdrant_url: str,
+        get_embedding_manager,
+        normalize_project_name
+    ):
+        """Initialize reflection tools with dependencies."""
+        self.qdrant_client = qdrant_client
+        self.qdrant_url = qdrant_url
+        self.get_embedding_manager = get_embedding_manager
+        self.normalize_project_name = normalize_project_name
+
+    async def store_reflection(
+        self,
+        ctx: Context,
+        content: str,
+        tags: List[str] = []
+    ) -> str:
+        """Store an important insight or reflection for future reference."""
+
+        await ctx.debug(f"Storing reflection with {len(tags)} tags")
+
+        try:
+            # Determine collection name based on embedding type
+            embedding_manager = self.get_embedding_manager()
+            embedding_type = "local" if embedding_manager.prefer_local else "voyage"
+            collection_name = f"reflections_{embedding_type}"
+
+            # Ensure reflections collection exists
+            try:
+                await self.qdrant_client.get_collection(collection_name)
+                await ctx.debug(f"Using existing {collection_name} collection")
+            except Exception:
+                # Collection doesn't exist, create it
+                await ctx.debug(f"Creating {collection_name} collection")
+
+                # Determine embedding dimensions
+                embedding_dim = embedding_manager.get_vector_dimension()
+
+                await self.qdrant_client.create_collection(
+                    collection_name=collection_name,
+                    vectors_config=VectorParams(
+                        size=embedding_dim,
+                        distance=Distance.COSINE
+                    )
+                )
+            
+            # Generate embedding for the reflection
+            embedding_manager = self.get_embedding_manager()
+            embedding = await embedding_manager.generate_embedding(content)
+            
+            # Create unique ID
+            reflection_id = hashlib.md5(f"{content}{datetime.now().isoformat()}".encode()).hexdigest()
+            
+            # Prepare metadata
+            metadata = {
+                "content": content,
+                "tags": tags,
+                "timestamp": datetime.now(timezone.utc).isoformat(),
+                "type": "reflection"
+            }
+            
+            # Store in Qdrant
+            await self.qdrant_client.upsert(
+                collection_name=collection_name,
+                points=[
+                    PointStruct(
+                        id=reflection_id,
+                        vector=embedding,
+                        payload=metadata
+                    )
+                ]
+            )
+            
+            await ctx.debug(f"Stored reflection with ID {reflection_id}")
+            
+            return f"""Reflection stored successfully.
+ID: {reflection_id}
+Tags: {', '.join(tags) if tags else 'none'}
+Timestamp: {metadata['timestamp']}"""
+            
+        except Exception as e:
+            logger.error(f"Failed to store reflection: {e}", exc_info=True)
+            return f"Failed to store reflection: {str(e)}"
+
+    async def get_full_conversation(
+        self,
+        ctx: Context,
+        conversation_id: str,
+        project: Optional[str] = None
+    ) -> str:
+        """Get the full JSONL conversation file path for a conversation ID.
+        This allows agents to read complete conversations instead of truncated excerpts."""
+        
+        await ctx.debug(f"Getting full conversation for ID: {conversation_id}, project: {project}")
+        
+        try:
+            # Base path for conversations
+            base_path = Path.home() / '.claude' / 'projects'
+            
+            # If project is specified, try to find it in that project
+            if project:
+                # Normalize project name for path matching
+                project_normalized = self.normalize_project_name(project)
+                
+                # Look for project directories that match
+                for project_dir in base_path.glob('*'):
+                    if project_normalized in project_dir.name.lower():
+                        # Look for JSONL files in this project
+                        for jsonl_file in project_dir.glob('*.jsonl'):
+                            # Check if filename matches conversation_id (with or without .jsonl)
+                            if conversation_id in jsonl_file.stem or conversation_id == jsonl_file.stem:
+                                await ctx.debug(f"Found conversation by filename in {jsonl_file}")
+                                return f"""<conversation_file>
+<conversation_id>{conversation_id}</conversation_id>
+<file_path>{str(jsonl_file)}</file_path>
+<project>{project_dir.name}</project>
+<message>Use the Read tool with this file path to read the complete conversation.</message>
+</conversation_file>"""
+            
+            # If not found in specific project or no project specified, search all
+            await ctx.debug("Searching all projects for conversation")
+            for project_dir in base_path.glob('*'):
+                for jsonl_file in project_dir.glob('*.jsonl'):
+                    # Check if filename matches conversation_id (with or without .jsonl)
+                    if conversation_id in jsonl_file.stem or conversation_id == jsonl_file.stem:
+                        await ctx.debug(f"Found conversation by filename in {jsonl_file}")
+                        return f"""<conversation_file>
+<conversation_id>{conversation_id}</conversation_id>
+<file_path>{str(jsonl_file)}</file_path>
+<project>{project_dir.name}</project>
+<message>Use the Read tool with this file path to read the complete conversation.</message>
+</conversation_file>"""
+            
+            # Not found
+            return f"""<conversation_file>
+<error>Conversation ID '{conversation_id}' not found in any project.</error>
+<suggestion>The conversation may not have been imported yet, or the ID may be incorrect.</suggestion>
+</conversation_file>"""
+            
+        except Exception as e:
+            logger.error(f"Failed to get conversation file: {e}", exc_info=True)
+            return f"""<conversation_file>
+<error>Failed to locate conversation: {str(e)}</error>
+</conversation_file>"""
+
+
+def register_reflection_tools(
+    mcp,
+    qdrant_client: AsyncQdrantClient,
+    qdrant_url: str,
+    get_embedding_manager,
+    normalize_project_name
+):
+    """Register reflection tools with the MCP server."""
+    
+    tools = ReflectionTools(
+        qdrant_client,
+        qdrant_url,
+        get_embedding_manager,
+        normalize_project_name
+    )
+    
+    @mcp.tool()
+    async def store_reflection(
+        ctx: Context,
+        content: str = Field(description="The insight or reflection to store"),
+        tags: List[str] = Field(default=[], description="Tags to categorize this reflection")
+    ) -> str:
+        """Store an important insight or reflection for future reference."""
+        return await tools.store_reflection(ctx, content, tags)
+    
+    @mcp.tool()
+    async def get_full_conversation(
+        ctx: Context,
+        conversation_id: str = Field(description="The conversation ID from search results (cid)"),
+        project: Optional[str] = Field(default=None, description="Optional project name to help locate the file")
+    ) -> str:
+        """Get the full JSONL conversation file path for a conversation ID.
+        This allows agents to read complete conversations instead of truncated excerpts."""
+        return await tools.get_full_conversation(ctx, conversation_id, project)
+    
+    logger.info("Reflection tools registered successfully")