claude-self-reflect 3.3.1 → 4.0.1

package/mcp-server/src/enhanced_tool_registry.py

@@ -0,0 +1,407 @@
+"""Enhanced tool registry with improved descriptions for better tool selection.
+
+This module provides enhanced tool registration with:
+1. csr_ namespace prefix for all tools
+2. Explicit "when to use" guidance in descriptions
+3. Response format flexibility (concise/detailed)
+4. Better tool grouping and discoverability
+"""
+
+from typing import Optional, List, Literal
+from fastmcp import Context
+from pydantic import Field
+
+
+def register_enhanced_search_tools(mcp, tools):
+    """Register search tools with enhanced descriptions for better selection rates."""
+
+    # Primary search tool - most commonly needed
+    @mcp.tool(name="csr_reflect_on_past")
+    async def csr_reflect_on_past(
+        ctx: Context,
+        query: str = Field(
+            description="The search query to find semantically similar conversations"
+        ),
+        limit: int = Field(
+            default=5,
+            description="Maximum number of results to return"
+        ),
+        min_score: float = Field(
+            default=0.3,
+            description="Minimum similarity score (0-1)"
+        ),
+        use_decay: int = Field(
+            default=-1,
+            description="Apply time-based decay: 1=enable, 0=disable, -1=use environment default"
+        ),
+        project: Optional[str] = Field(
+            default=None,
+            description="Search specific project only. Use 'all' to search across all projects"
+        ),
+        mode: str = Field(
+            default="full",
+            description="Search mode: 'full' (all results), 'quick' (count only), 'summary' (insights)"
+        ),
+        response_format: Literal["concise", "detailed", "xml"] = Field(
+            default="xml",
+            description="Output format: 'concise' for brief results, 'detailed' for full context, 'xml' for structured"
+        )
+    ) -> str:
+        """Search past Claude conversations semantically to find relevant context.
+
+        WHEN TO USE THIS TOOL:
+        - User asks "what did we discuss about X?" or "find conversations about Y"
+        - You need context from previous work on similar problems
+        - User mentions "remember when" or "last time we"
+        - Debugging issues that may have been solved before
+        - Finding implementation patterns used in the project
+
+        EXAMPLES THAT TRIGGER THIS TOOL:
+        - "What did we work on with Docker last week?"
+        - "Find all conversations about authentication"
+        - "How did we solve the memory leak issue?"
+        - "Search for discussions about database optimization"
+
+        This is the PRIMARY tool for conversation memory - use it liberally!
+        """
+        # Map response_format to existing parameters
+        brief = response_format == "concise"
+        include_raw = response_format == "detailed"
+
+        return await tools.reflect_on_past(
+            ctx, query, limit, min_score, use_decay,
+            project, mode, brief, include_raw, response_format="xml"
+        )
+
+    # Quick existence check - for fast validation
+    @mcp.tool(name="csr_quick_check")
+    async def csr_quick_check(
+        ctx: Context,
+        query: str = Field(
+            description="Topic or concept to check for existence"
+        ),
+        min_score: float = Field(
+            default=0.3,
+            description="Minimum similarity score (0-1)"
+        ),
+        project: Optional[str] = Field(
+            default=None,
+            description="Search specific project only"
+        )
+    ) -> str:
+        """Quick check if a topic was discussed before (returns count + top match only).
+
+        WHEN TO USE THIS TOOL:
+        - User asks "have we discussed X?" or "is there anything about Y?"
+        - You need a yes/no answer about topic existence
+        - Checking if a problem was encountered before
+        - Validating if a concept is familiar to the project
+
+        EXAMPLES THAT TRIGGER THIS TOOL:
+        - "Have we talked about WebSockets?"
+        - "Is there any discussion about React hooks?"
+        - "Did we ever implement caching?"
+
+        Much faster than full search - use for existence checks!
+        """
+        return await tools.quick_search(ctx, query, min_score, project)
+
+    # Time-based search - for recent work
+    @mcp.tool(name="csr_recent_work")
+    async def csr_recent_work(
+        ctx: Context,
+        limit: int = Field(
+            default=10,
+            description="Number of recent conversations to return"
+        ),
+        group_by: str = Field(
+            default="conversation",
+            description="Group by 'conversation', 'day', or 'session'"
+        ),
+        include_reflections: bool = Field(
+            default=True,
+            description="Include stored reflections"
+        ),
+        project: Optional[str] = Field(
+            default=None,
+            description="Specific project or 'all' for cross-project"
+        ),
+        response_format: Literal["concise", "detailed"] = Field(
+            default="concise",
+            description="Output verbosity level"
+        )
+    ) -> str:
+        """Get recent work conversations to understand current context.
+
+        WHEN TO USE THIS TOOL:
+        - User asks "what did we work on recently?" or "what were we doing?"
+        - Starting a new session and need context
+        - User says "continue from where we left off"
+        - Reviewing progress over time periods
+
+        EXAMPLES THAT TRIGGER THIS TOOL:
+        - "What did we work on yesterday?"
+        - "Show me the last 5 things we discussed"
+        - "What have I been working on this week?"
+        - "Let's continue from last time"
+
+        Essential for session continuity and context awareness!
+        """
+        return await tools.get_recent_work(ctx, group_by, limit, include_reflections, project)
+
+    # Time-constrained semantic search
+    @mcp.tool(name="csr_search_by_time")
+    async def csr_search_by_time(
+        ctx: Context,
+        query: str = Field(
+            description="Semantic search query"
+        ),
+        time_range: Optional[str] = Field(
+            default=None,
+            description="Natural language time like 'last week', 'yesterday'"
+        ),
+        since: Optional[str] = Field(
+            default=None,
+            description="ISO timestamp or relative time"
+        ),
+        until: Optional[str] = Field(
+            default=None,
+            description="ISO timestamp or relative time"
+        ),
+        limit: int = Field(
+            default=10,
+            description="Maximum number of results"
+        ),
+        min_score: float = Field(
+            default=0.3,
+            description="Minimum similarity score"
+        ),
+        project: Optional[str] = Field(
+            default=None,
+            description="Specific project or 'all'"
+        )
+    ) -> str:
+        """Search with time constraints for time-specific queries.
+
+        WHEN TO USE THIS TOOL:
+        - Query includes time references like "last week", "yesterday", "this month"
+        - User wants recent occurrences of a topic
+        - Debugging issues that started at a specific time
+        - Finding when something was first discussed
+
+        EXAMPLES THAT TRIGGER THIS TOOL:
+        - "Docker errors from last week"
+        - "What did we discuss about testing yesterday?"
+        - "Authentication problems in the past 3 days"
+        - "Recent conversations about performance"
+
+        Combines semantic search with temporal filtering!
+        """
+        return await tools.search_by_recency(
+            ctx, query, limit, min_score, project,
+            since, until, time_range
+        )
+
+    # File-based search - for code archaeology
+    @mcp.tool(name="csr_search_by_file")
+    async def csr_search_by_file(
+        ctx: Context,
+        file_path: str = Field(
+            description="File path to search for (absolute or relative)"
+        ),
+        limit: int = Field(
+            default=10,
+            description="Maximum number of results"
+        ),
+        project: Optional[str] = Field(
+            default=None,
+            description="Search specific project only"
+        )
+    ) -> str:
+        """Find all conversations that analyzed or modified a specific file.
+
+        WHEN TO USE THIS TOOL:
+        - User asks "when did we modify X file?" or "who worked on Y?"
+        - Investigating file history beyond git
+        - Understanding why changes were made to a file
+        - Finding discussions about specific code files
+
+        EXAMPLES THAT TRIGGER THIS TOOL:
+        - "When did we last modify server.py?"
+        - "Find all discussions about package.json"
+        - "What changes were made to the auth module?"
+        - "Who worked on the database schema?"
+
+        Perfect for code archaeology and understanding file evolution!
+        """
+        return await tools.search_by_file(ctx, file_path, limit, project)
+
+    # Concept-based search - for thematic queries
+    @mcp.tool(name="csr_search_by_concept")
+    async def csr_search_by_concept(
+        ctx: Context,
+        concept: str = Field(
+            description="Development concept (e.g., 'security', 'testing', 'performance')"
+        ),
+        limit: int = Field(
+            default=10,
+            description="Maximum number of results"
+        ),
+        include_files: bool = Field(
+            default=True,
+            description="Include file information"
+        ),
+        project: Optional[str] = Field(
+            default=None,
+            description="Search specific project only"
+        )
+    ) -> str:
+        """Search for conversations about specific development concepts or themes.
+
+        WHEN TO USE THIS TOOL:
+        - User asks about broad topics like "security", "testing", "performance"
+        - Looking for all discussions on a technical theme
+        - Gathering knowledge about how a concept is handled
+        - Finding patterns across multiple conversations
+
+        EXAMPLES THAT TRIGGER THIS TOOL:
+        - "Show me all security-related discussions"
+        - "Find conversations about testing strategies"
+        - "What have we discussed about performance?"
+        - "Look for Docker-related conversations"
+
+        Ideal for thematic analysis and knowledge gathering!
+        """
+        return await tools.search_by_concept(ctx, concept, limit, project, include_files)
+
+    # Insight storage - for knowledge persistence
+    @mcp.tool(name="csr_store_insight")
+    async def csr_store_insight(
+        ctx: Context,
+        content: str = Field(
+            description="The insight, solution, or learning to store"
+        ),
+        tags: List[str] = Field(
+            default=[],
+            description="Tags for categorization (e.g., ['docker', 'debugging'])"
+        )
+    ) -> str:
+        """Store important insights, solutions, or learnings for future reference.
+
+        WHEN TO USE THIS TOOL:
+        - User says "remember this" or "store this solution"
+        - After solving a complex problem
+        - When discovering important patterns or gotchas
+        - User provides valuable configuration or setup info
+        - After successful debugging sessions
+
+        EXAMPLES THAT TRIGGER THIS TOOL:
+        - "Remember this Docker configuration for next time"
+        - "Store this solution for the auth problem"
+        - "Save this debugging technique"
+        - "This is important - the API key goes in .env"
+
+        Critical for building institutional memory!
+        """
+        return await tools.store_reflection(ctx, content, tags)
+
+    # Aggregated insights - for analysis
+    @mcp.tool(name="csr_search_insights")
+    async def csr_search_insights(
+        ctx: Context,
+        query: str = Field(
+            description="Topic to analyze across conversations"
+        ),
+        project: Optional[str] = Field(
+            default=None,
+            description="Search specific project only"
+        )
+    ) -> str:
+        """Get aggregated insights and patterns from search results.
+
+        WHEN TO USE THIS TOOL:
+        - User wants patterns or trends, not individual results
+        - Analyzing how a topic evolved over time
+        - Understanding common themes across conversations
+        - Getting a high-level view without details
+
+        EXAMPLES THAT TRIGGER THIS TOOL:
+        - "What patterns do we see in error handling?"
+        - "Summarize our authentication discussions"
+        - "What are the common Docker issues we face?"
+        - "Give me insights about our testing approach"
+
+        Provides analysis, not just search results!
+        """
+        return await tools.search_summary(ctx, query, project)
+
+    # Pagination support - for deep dives
+    @mcp.tool(name="csr_get_more")
+    async def csr_get_more(
+        ctx: Context,
+        query: str = Field(
+            description="The original search query"
+        ),
+        offset: int = Field(
+            default=3,
+            description="Number of results to skip"
+        ),
+        limit: int = Field(
+            default=3,
+            description="Number of additional results"
+        ),
+        min_score: float = Field(
+            default=0.3,
+            description="Minimum similarity score"
+        ),
+        project: Optional[str] = Field(
+            default=None,
+            description="Search specific project only"
+        )
+    ) -> str:
+        """Get additional search results for paginated exploration.
+
+        WHEN TO USE THIS TOOL:
+        - User says "show me more" after a search
+        - Initial results weren't sufficient
+        - Deep diving into a topic
+        - User wants comprehensive coverage
+
+        EXAMPLES THAT TRIGGER THIS TOOL:
+        - "Show me more results"
+        - "What else is there?"
+        - "Keep searching"
+        - "I need more examples"
+
+        Use after initial search when more context is needed!
+        """
+        return await tools.get_more_results(ctx, query, offset, limit, min_score, project)
+
+    # Full conversation retrieval
+    @mcp.tool(name="csr_get_full_conversation")
+    async def csr_get_full_conversation(
+        ctx: Context,
+        conversation_id: str = Field(
+            description="Conversation ID from search results (cid field)"
+        ),
+        project: Optional[str] = Field(
+            default=None,
+            description="Optional project name to help locate the file"
+        )
+    ) -> str:
+        """Get the full conversation file path to read complete context.
+
+        WHEN TO USE THIS TOOL:
+        - Search result was truncated but needs full context
+        - User wants to see the entire conversation
+        - Debugging requires complete conversation history
+        - Following up on a specific conversation ID
+
+        EXAMPLES THAT TRIGGER THIS TOOL:
+        - "Show me the full conversation for cid_12345"
+        - "I need to see everything from that discussion"
+        - "Get the complete context for that result"
+
+        Returns file path for agents to read complete conversations!
+        """
+        return await tools.get_full_conversation(ctx, conversation_id, project)

package/mcp-server/src/mode_switch_tool.py

@@ -0,0 +1,181 @@
+"""Runtime mode switching tool for embedding models."""
+
+import os
+import logging
+from typing import Literal
+from fastmcp import Context
+from pydantic import Field
+
+logger = logging.getLogger(__name__)
+
+
+class ModeSwitcher:
+    """Handles runtime switching between embedding modes."""
+
+    def __init__(self, get_embedding_manager):
+        """Initialize with embedding manager getter."""
+        self.get_embedding_manager = get_embedding_manager
+
+    async def switch_mode(
+        self,
+        ctx: Context,
+        mode: Literal["local", "cloud"]
+    ) -> str:
+        """Switch between local and cloud embedding modes at runtime."""
+
+        await ctx.debug(f"Switching to {mode} mode...")
+
+        try:
+            # Get the current embedding manager
+            manager = self.get_embedding_manager()
+
+            # Store current state
+            old_mode = manager.model_type
+            old_prefer_local = manager.prefer_local
+
+            # Update configuration based on requested mode
+            if mode == "local":
+                # Switch to local mode
+                manager.prefer_local = True
+                # Clear voyage key to force local
+                manager.voyage_key = None
+
+                # Reinitialize with local preference
+                if not manager.local_model:
+                    success = manager.try_initialize_local()
+                    if not success:
+                        return "❌ Failed to initialize local model"
+
+                # Update default model type
+                manager.model_type = 'local'
+
+                await ctx.debug("Switched to LOCAL mode (FastEmbed, 384 dimensions)")
+
+            elif mode == "cloud":
+                # Switch to cloud mode
+                # First check if we have a Voyage key
+                voyage_key = os.getenv('VOYAGE_KEY') or os.getenv('VOYAGE_KEY-2')
+                if not voyage_key:
+                    # Try to load from .env file
+                    from pathlib import Path
+                    from dotenv import load_dotenv
+                    env_path = Path(__file__).parent.parent.parent / '.env'
+                    load_dotenv(env_path, override=True)
+                    voyage_key = os.getenv('VOYAGE_KEY') or os.getenv('VOYAGE_KEY-2')
+
+                if not voyage_key:
+                    return "❌ Cannot switch to cloud mode: VOYAGE_KEY not found in environment or .env file"
+
+                manager.prefer_local = False
+                manager.voyage_key = voyage_key
+
+                # Reinitialize Voyage client
+                if not manager.voyage_client:
+                    success = manager.try_initialize_voyage()
+                    if not success:
+                        # Restore previous state
+                        manager.prefer_local = old_prefer_local
+                        manager.model_type = old_mode
+                        return "❌ Failed to initialize Voyage AI client"
+
+                # Update default model type
+                manager.model_type = 'voyage'
+
+                await ctx.debug("Switched to CLOUD mode (Voyage AI, 1024 dimensions)")
+
+            # Log the switch
+            logger.info(f"Mode switched from {old_mode} to {manager.model_type}")
+
+            # Prepare detailed response
+            return f"""✅ Successfully switched to {mode.upper()} mode!
+
+**Previous Configuration:**
+- Mode: {old_mode}
+- Prefer Local: {old_prefer_local}
+
+**New Configuration:**
+- Mode: {manager.model_type}
+- Prefer Local: {manager.prefer_local}
+- Vector Dimensions: {manager.get_vector_dimension()}
+- Has Voyage Key: {bool(manager.voyage_key)}
+
+**Important Notes:**
+- New reflections will go to: reflections_{manager.model_type}
+- Existing collections remain unchanged
+- No restart required! 🎉
+
+**Next Steps:**
+- Use `store_reflection` to test the new mode
+- Use `reflect_on_past` to search across all collections"""
+
+        except Exception as e:
+            logger.error(f"Failed to switch mode: {e}", exc_info=True)
+            return f"❌ Failed to switch mode: {str(e)}"
+
+    async def get_current_mode(self, ctx: Context) -> str:
+        """Get the current embedding mode and configuration."""
+
+        try:
+            manager = self.get_embedding_manager()
+
+            # Check actual model availability
+            local_available = manager.local_model is not None
+            voyage_available = manager.voyage_client is not None
+
+            return f"""📊 Current Embedding Configuration:
+
+**Active Mode:** {manager.model_type.upper()}
+**Vector Dimensions:** {manager.get_vector_dimension()}
+
+**Configuration:**
+- Prefer Local: {manager.prefer_local}
+- Has Voyage Key: {bool(manager.voyage_key)}
+
+**Available Models:**
+- Local (FastEmbed): {'✅ Initialized' if local_available else '❌ Not initialized'}
+- Cloud (Voyage AI): {'✅ Initialized' if voyage_available else '❌ Not initialized'}
+
+**Collection Names:**
+- Reflections: reflections_{manager.model_type}
+- Conversations: [project]_{manager.model_type}
+
+**Environment:**
+- PREFER_LOCAL_EMBEDDINGS: {os.getenv('PREFER_LOCAL_EMBEDDINGS', 'not set')}
+- VOYAGE_KEY: {'set' if manager.voyage_key else 'not set'}"""
+
+        except Exception as e:
+            logger.error(f"Failed to get current mode: {e}", exc_info=True)
+            return f"❌ Failed to get current mode: {str(e)}"
+
+
+def register_mode_switch_tool(mcp, get_embedding_manager):
+    """Register the mode switching tool with the MCP server."""
+
+    switcher = ModeSwitcher(get_embedding_manager)
+
+    @mcp.tool()
+    async def switch_embedding_mode(
+        ctx: Context,
+        mode: Literal["local", "cloud"] = Field(
+            description="Target embedding mode: 'local' for FastEmbed (384 dim), 'cloud' for Voyage AI (1024 dim)"
+        )
+    ) -> str:
+        """Switch between local and cloud embedding modes at runtime without restarting the MCP server.
+
+        This allows dynamic switching between:
+        - LOCAL mode: FastEmbed with 384 dimensions (privacy-first, no API calls)
+        - CLOUD mode: Voyage AI with 1024 dimensions (better quality, requires API key)
+
+        No restart required! The change takes effect immediately for all new operations.
+        """
+        return await switcher.switch_mode(ctx, mode)
+
+    @mcp.tool()
+    async def get_embedding_mode(ctx: Context) -> str:
+        """Get the current embedding mode configuration and status.
+
+        Shows which mode is active, available models, and collection naming.
+        """
+        return await switcher.get_current_mode(ctx)
+
+    logger.info("Mode switching tools registered successfully")

package/mcp-server/src/parallel_search.py

@@ -70,18 +70,27 @@ async def search_single_collection(
             # This code path is intentionally disabled
             pass
         else:
+            # SECURITY FIX: Reduce memory multiplier to prevent OOM
+            from .security_patches import MemoryOptimizer
+            safe_limit = MemoryOptimizer.calculate_safe_limit(limit, 1.5) if should_use_decay else limit
+
             # Standard search without native decay or client-side decay
             search_results = await qdrant_client.search(
                 collection_name=collection_name,
                 query_vector=query_embedding,
-                limit=limit * 3 if should_use_decay else limit,  # Get more results for client-side decay
+                limit=safe_limit,  # Use safe limit to prevent memory explosion
                 score_threshold=min_score if not should_use_decay else 0.0,
                 with_payload=True
             )
 
+            # CRITICAL FIX: Handle None search results (cloud mode issue)
+            if search_results is None:
+                logger.warning(f"Search returned None for collection {collection_name}")
+                search_results = []
+
             # Debug: Log search results
             logger.debug(f"Search of {collection_name} returned {len(search_results)} results")
-            
+
             if should_use_decay and not USE_NATIVE_DECAY:
                 # Apply client-side decay
                 await ctx.debug(f"Using CLIENT-SIDE decay for {collection_name}")
@@ -292,8 +301,9 @@ async def parallel_search_collections(
     
     for result in search_results:
         if isinstance(result, Exception):
-            # Handle exceptions from gather
-            logger.error(f"Search task failed: {result}")
+            # SECURITY FIX: Proper exception logging with context
+            from .security_patches import ExceptionLogger
+            ExceptionLogger.log_exception(result, "parallel_search_task")
             continue
         
         collection_name, results, timing = result

package/mcp-server/src/project_resolver.py

@@ -219,11 +219,27 @@ class ProjectResolver:
                     logger.debug(f"Failed to scroll {coll_name}: {e}")
                     continue
         
+        # Add appropriate reflection collections based on the found conversation collections
+        # If we found _local collections, add reflections_local
+        # If we found _voyage collections, add reflections_voyage
+        reflection_collections = set()
+        for coll in matching_collections:
+            if coll.endswith('_local') and 'reflections_local' in collection_names:
+                reflection_collections.add('reflections_local')
+            elif coll.endswith('_voyage') and 'reflections_voyage' in collection_names:
+                reflection_collections.add('reflections_voyage')
+
+        # Also check for the legacy 'reflections' collection
+        if 'reflections' in collection_names:
+            reflection_collections.add('reflections')
+
+        matching_collections.update(reflection_collections)
+
         # Cache the result with TTL
         result = list(matching_collections)
         self._cache[user_project_name] = matching_collections
         self._cache_ttl[user_project_name] = time()
-        
+
         return result
     
     def _get_collection_names(self, force_refresh: bool = False) -> List[str]:
@@ -244,7 +260,9 @@ class ProjectResolver:
         # Fetch fresh collection list
         try:
             all_collections = self.client.get_collections().collections
-            collection_names = [c.name for c in all_collections if c.name.startswith('conv_')]
+            # Include both conversation collections and reflection collections
+            collection_names = [c.name for c in all_collections
+                              if c.name.startswith('conv_') or c.name.startswith('reflections')]
             
             # Update cache
             self._collections_cache = collection_names