claude-self-reflect 3.3.0 → 4.0.0

package/mcp-server/src/mode_switch_tool.py

@@ -0,0 +1,181 @@
+"""Runtime mode switching tool for embedding models."""
+
+import os
+import logging
+from typing import Literal
+from fastmcp import Context
+from pydantic import Field
+
+logger = logging.getLogger(__name__)
+
+
+class ModeSwitcher:
+    """Handles runtime switching between embedding modes."""
+
+    def __init__(self, get_embedding_manager):
+        """Initialize with embedding manager getter."""
+        self.get_embedding_manager = get_embedding_manager
+
+    async def switch_mode(
+        self,
+        ctx: Context,
+        mode: Literal["local", "cloud"]
+    ) -> str:
+        """Switch between local and cloud embedding modes at runtime."""
+
+        await ctx.debug(f"Switching to {mode} mode...")
+
+        try:
+            # Get the current embedding manager
+            manager = self.get_embedding_manager()
+
+            # Store current state
+            old_mode = manager.model_type
+            old_prefer_local = manager.prefer_local
+
+            # Update configuration based on requested mode
+            if mode == "local":
+                # Switch to local mode
+                manager.prefer_local = True
+                # Clear voyage key to force local
+                manager.voyage_key = None
+
+                # Reinitialize with local preference
+                if not manager.local_model:
+                    success = manager.try_initialize_local()
+                    if not success:
+                        return "❌ Failed to initialize local model"
+
+                # Update default model type
+                manager.model_type = 'local'
+
+                await ctx.debug("Switched to LOCAL mode (FastEmbed, 384 dimensions)")
+
+            elif mode == "cloud":
+                # Switch to cloud mode
+                # First check if we have a Voyage key
+                voyage_key = os.getenv('VOYAGE_KEY') or os.getenv('VOYAGE_KEY-2')
+                if not voyage_key:
+                    # Try to load from .env file
+                    from pathlib import Path
+                    from dotenv import load_dotenv
+                    env_path = Path(__file__).parent.parent.parent / '.env'
+                    load_dotenv(env_path, override=True)
+                    voyage_key = os.getenv('VOYAGE_KEY') or os.getenv('VOYAGE_KEY-2')
+
+                if not voyage_key:
+                    return "❌ Cannot switch to cloud mode: VOYAGE_KEY not found in environment or .env file"
+
+                manager.prefer_local = False
+                manager.voyage_key = voyage_key
+
+                # Reinitialize Voyage client
+                if not manager.voyage_client:
+                    success = manager.try_initialize_voyage()
+                    if not success:
+                        # Restore previous state
+                        manager.prefer_local = old_prefer_local
+                        manager.model_type = old_mode
+                        return "❌ Failed to initialize Voyage AI client"
+
+                # Update default model type
+                manager.model_type = 'voyage'
+
+                await ctx.debug("Switched to CLOUD mode (Voyage AI, 1024 dimensions)")
+
+            # Log the switch
+            logger.info(f"Mode switched from {old_mode} to {manager.model_type}")
+
+            # Prepare detailed response
+            return f"""✅ Successfully switched to {mode.upper()} mode!
+
+**Previous Configuration:**
+- Mode: {old_mode}
+- Prefer Local: {old_prefer_local}
+
+**New Configuration:**
+- Mode: {manager.model_type}
+- Prefer Local: {manager.prefer_local}
+- Vector Dimensions: {manager.get_vector_dimension()}
+- Has Voyage Key: {bool(manager.voyage_key)}
+
+**Important Notes:**
+- New reflections will go to: reflections_{manager.model_type}
+- Existing collections remain unchanged
+- No restart required! 🎉
+
+**Next Steps:**
+- Use `store_reflection` to test the new mode
+- Use `reflect_on_past` to search across all collections"""
+
+        except Exception as e:
+            logger.error(f"Failed to switch mode: {e}", exc_info=True)
+            return f"❌ Failed to switch mode: {str(e)}"
+
+    async def get_current_mode(self, ctx: Context) -> str:
+        """Get the current embedding mode and configuration."""
+
+        try:
+            manager = self.get_embedding_manager()
+
+            # Check actual model availability
+            local_available = manager.local_model is not None
+            voyage_available = manager.voyage_client is not None
+
+            return f"""📊 Current Embedding Configuration:
+
+**Active Mode:** {manager.model_type.upper()}
+**Vector Dimensions:** {manager.get_vector_dimension()}
+
+**Configuration:**
+- Prefer Local: {manager.prefer_local}
+- Has Voyage Key: {bool(manager.voyage_key)}
+
+**Available Models:**
+- Local (FastEmbed): {'✅ Initialized' if local_available else '❌ Not initialized'}
+- Cloud (Voyage AI): {'✅ Initialized' if voyage_available else '❌ Not initialized'}
+
+**Collection Names:**
+- Reflections: reflections_{manager.model_type}
+- Conversations: [project]_{manager.model_type}
+
+**Environment:**
+- PREFER_LOCAL_EMBEDDINGS: {os.getenv('PREFER_LOCAL_EMBEDDINGS', 'not set')}
+- VOYAGE_KEY: {'set' if manager.voyage_key else 'not set'}"""
+
+        except Exception as e:
+            logger.error(f"Failed to get current mode: {e}", exc_info=True)
+            return f"❌ Failed to get current mode: {str(e)}"
+
+
+def register_mode_switch_tool(mcp, get_embedding_manager):
+    """Register the mode switching tool with the MCP server."""
+
+    switcher = ModeSwitcher(get_embedding_manager)
+
+    @mcp.tool()
+    async def switch_embedding_mode(
+        ctx: Context,
+        mode: Literal["local", "cloud"] = Field(
+            description="Target embedding mode: 'local' for FastEmbed (384 dim), 'cloud' for Voyage AI (1024 dim)"
+        )
+    ) -> str:
+        """Switch between local and cloud embedding modes at runtime without restarting the MCP server.
+
+        This allows dynamic switching between:
+        - LOCAL mode: FastEmbed with 384 dimensions (privacy-first, no API calls)
+        - CLOUD mode: Voyage AI with 1024 dimensions (better quality, requires API key)
+
+        No restart required! The change takes effect immediately for all new operations.
+        """
+        return await switcher.switch_mode(ctx, mode)
+
+    @mcp.tool()
+    async def get_embedding_mode(ctx: Context) -> str:
+        """Get the current embedding mode configuration and status.
+
+        Shows which mode is active, available models, and collection naming.
+        """
+        return await switcher.get_current_mode(ctx)
+
+    logger.info("Mode switching tools registered successfully")

package/mcp-server/src/parallel_search.py

@@ -64,90 +64,21 @@ async def search_single_collection(
         DECAY_SCALE_DAYS = constants.get('DECAY_SCALE_DAYS', 90)
         DECAY_WEIGHT = constants.get('DECAY_WEIGHT', 0.3)
         
-        if should_use_decay and USE_NATIVE_DECAY and NATIVE_DECAY_AVAILABLE:
-            # Use native Qdrant decay with newer API
-            await ctx.debug(f"Using NATIVE Qdrant decay (new API) for {collection_name}")
-            
-            half_life_seconds = DECAY_SCALE_DAYS * 24 * 60 * 60
-            
-            # Build query using Qdrant's Fusion and RankFusion
-            fusion_query = models.Fusion(
-                fusion=models.RankFusion.RRF,
-                queries=[
-                    # Semantic similarity query
-                    models.NearestQuery(
-                        nearest=query_embedding,
-                        score_threshold=min_score
-                    ),
-                    # Time decay query using context pair
-                    models.ContextQuery(
-                        context=[
-                            models.ContextPair(
-                                positive=models.DiscoverQuery(
-                                    target=query_embedding,
-                                    context=[
-                                        models.ContextPair(
-                                            positive=models.DatetimeRange(
-                                                gt=datetime.now().isoformat(),
-                                                lt=(datetime.now().timestamp() + half_life_seconds)
-                                            )
-                                        )
-                                    ]
-                                )
-                            )
-                        ]
-                    )
-                ]
-            )
-            
-            # Execute search with native decay
-            search_results = await qdrant_client.query_points(
-                collection_name=collection_name,
-                query=fusion_query,
-                limit=limit,
-                with_payload=True
-            )
-            
-            # Process results
-            for point in search_results.points:
-                # Process each point and add to results
-                raw_timestamp = point.payload.get('timestamp', datetime.now().isoformat())
-                clean_timestamp = raw_timestamp.replace('Z', '+00:00') if raw_timestamp.endswith('Z') else raw_timestamp
-                
-                point_project = point.payload.get('project', collection_name.replace('conv_', '').replace('_voyage', '').replace('_local', ''))
-                
-                # Apply project filtering
-                if target_project != 'all' and not is_reflection_collection:
-                    if point_project != target_project:
-                        normalized_target = target_project.replace('-', '_')
-                        normalized_point = point_project.replace('-', '_')
-                        if not (normalized_point == normalized_target or 
-                                point_project.endswith(f"/{target_project}") or
-                                point_project.endswith(f"-{target_project}") or
-                                normalized_point.endswith(f"_{normalized_target}") or
-                                normalized_point.endswith(f"/{normalized_target}")):
-                            continue
-                
-                # Create SearchResult
-                search_result = {
-                    'id': str(point.id),
-                    'score': point.score,
-                    'timestamp': clean_timestamp,
-                    'role': point.payload.get('start_role', point.payload.get('role', 'unknown')),
-                    'excerpt': (point.payload.get('text', '')[:350] + '...' 
-                               if len(point.payload.get('text', '')) > 350 
-                               else point.payload.get('text', '')),
-                    'project_name': point_project,
-                    'payload': point.payload
-                }
-                results.append(search_result)
-                
+        # NOTE: Native decay API is not available in current Qdrant, fall back to client-side
+        # The Fusion/RankFusion API was experimental and removed, always use client-side decay
+        if should_use_decay and False:  # Disabled until Qdrant provides stable decay API
+            # This code path is intentionally disabled
+            pass
         else:
+            # SECURITY FIX: Reduce memory multiplier to prevent OOM
+            from .security_patches import MemoryOptimizer
+            safe_limit = MemoryOptimizer.calculate_safe_limit(limit, 1.5) if should_use_decay else limit
+
             # Standard search without native decay or client-side decay
             search_results = await qdrant_client.search(
                 collection_name=collection_name,
                 query_vector=query_embedding,
-                limit=limit * 3 if should_use_decay else limit,  # Get more results for client-side decay
+                limit=safe_limit,  # Use safe limit to prevent memory explosion
                 score_threshold=min_score if not should_use_decay else 0.0,
                 with_payload=True
             )
@@ -220,17 +151,24 @@ async def search_single_collection(
                                 continue
                         logger.debug(f"Keeping point: project '{point_project}' matches target '{target_project}'")
                     
-                    # Create SearchResult
+                    # Create SearchResult with consistent structure
                     search_result = {
                         'id': str(point.id),
                         'score': adjusted_score,
                         'timestamp': clean_timestamp,
                         'role': point.payload.get('start_role', point.payload.get('role', 'unknown')),
-                        'excerpt': (point.payload.get('text', '')[:350] + '...' 
-                                   if len(point.payload.get('text', '')) > 350 
+                        'excerpt': (point.payload.get('text', '')[:350] + '...'
+                                   if len(point.payload.get('text', '')) > 350
                                    else point.payload.get('text', '')),
                         'project_name': point_project,
-                        'payload': point.payload
+                        'conversation_id': point.payload.get('conversation_id'),
+                        'base_conversation_id': point.payload.get('base_conversation_id'),
+                        'collection_name': collection_name,
+                        'raw_payload': point.payload,  # Renamed from 'payload' for consistency
+                        'code_patterns': point.payload.get('code_patterns'),
+                        'files_analyzed': point.payload.get('files_analyzed'),
+                        'tools_used': list(point.payload.get('tools_used', [])) if isinstance(point.payload.get('tools_used'), set) else point.payload.get('tools_used'),
+                        'concepts': point.payload.get('concepts')
                     }
                     results.append(search_result)
             else:
@@ -358,8 +296,9 @@ async def parallel_search_collections(
     
     for result in search_results:
         if isinstance(result, Exception):
-            # Handle exceptions from gather
-            logger.error(f"Search task failed: {result}")
+            # SECURITY FIX: Proper exception logging with context
+            from .security_patches import ExceptionLogger
+            ExceptionLogger.log_exception(result, "parallel_search_task")
             continue
         
         collection_name, results, timing = result

package/mcp-server/src/project_resolver.py

@@ -219,11 +219,27 @@ class ProjectResolver:
                     logger.debug(f"Failed to scroll {coll_name}: {e}")
                     continue
         
+        # Add appropriate reflection collections based on the found conversation collections
+        # If we found _local collections, add reflections_local
+        # If we found _voyage collections, add reflections_voyage
+        reflection_collections = set()
+        for coll in matching_collections:
+            if coll.endswith('_local') and 'reflections_local' in collection_names:
+                reflection_collections.add('reflections_local')
+            elif coll.endswith('_voyage') and 'reflections_voyage' in collection_names:
+                reflection_collections.add('reflections_voyage')
+
+        # Also check for the legacy 'reflections' collection
+        if 'reflections' in collection_names:
+            reflection_collections.add('reflections')
+
+        matching_collections.update(reflection_collections)
+
         # Cache the result with TTL
         result = list(matching_collections)
         self._cache[user_project_name] = matching_collections
         self._cache_ttl[user_project_name] = time()
-        
+
         return result
     
     def _get_collection_names(self, force_refresh: bool = False) -> List[str]:
@@ -244,7 +260,9 @@ class ProjectResolver:
         # Fetch fresh collection list
         try:
             all_collections = self.client.get_collections().collections
-            collection_names = [c.name for c in all_collections if c.name.startswith('conv_')]
+            # Include both conversation collections and reflection collections
+            collection_names = [c.name for c in all_collections
+                              if c.name.startswith('conv_') or c.name.startswith('reflections')]
             
             # Update cache
             self._collections_cache = collection_names

package/mcp-server/src/reflection_tools.py

@@ -44,9 +44,24 @@ class ReflectionTools:
         await ctx.debug(f"Storing reflection with {len(tags)} tags")
 
         try:
-            # Determine collection name based on embedding type
+            # Check runtime preference from environment
+            import os
+            prefer_local = os.getenv('PREFER_LOCAL_EMBEDDINGS', 'true').lower() == 'true'
+
             embedding_manager = self.get_embedding_manager()
-            embedding_type = "local" if embedding_manager.prefer_local else "voyage"
+
+            # Use embedding_manager's model_type which already respects preferences
+            embedding_type = embedding_manager.model_type
+
+            if embedding_type == "local":
+                await ctx.debug("Using LOCAL mode (FastEmbed, 384 dimensions)")
+            elif embedding_type == "voyage":
+                await ctx.debug("Using VOYAGE mode (Voyage AI, 1024 dimensions)")
+            else:
+                # Shouldn't happen but handle gracefully
+                embedding_type = "local" if embedding_manager.local_model else "voyage"
+                await ctx.debug(f"Using {embedding_type} mode (fallback)")
+
             collection_name = f"reflections_{embedding_type}"
 
             # Ensure reflections collection exists
@@ -57,8 +72,8 @@ class ReflectionTools:
                 # Collection doesn't exist, create it
                 await ctx.debug(f"Creating {collection_name} collection")
 
-                # Determine embedding dimensions
-                embedding_dim = embedding_manager.get_vector_dimension()
+                # Get embedding dimensions for the specific type
+                embedding_dim = embedding_manager.get_vector_dimension(force_type=embedding_type)
 
                 await self.qdrant_client.create_collection(
                     collection_name=collection_name,
@@ -67,13 +82,18 @@ class ReflectionTools:
                         distance=Distance.COSINE
                     )
                 )
+
+            # Generate embedding with the same forced type for consistency
+            embedding = await embedding_manager.generate_embedding(content, force_type=embedding_type)
+
+            # Guard against failed embeddings
+            if not embedding:
+                await ctx.debug("Failed to generate embedding for reflection")
+                return "Failed to store reflection: embedding generation failed"
             
-            # Generate embedding for the reflection
-            embedding_manager = self.get_embedding_manager()
-            embedding = await embedding_manager.generate_embedding(content)
-            
-            # Create unique ID
-            reflection_id = hashlib.md5(f"{content}{datetime.now().isoformat()}".encode()).hexdigest()
+            # SECURITY FIX: Use SHA-256 instead of MD5
+            from .security_patches import SecureHashGenerator
+            reflection_id = SecureHashGenerator.generate_id(f"{content}{datetime.now().isoformat()}")
             
             # Prepare metadata
             metadata = {
@@ -99,6 +119,7 @@ class ReflectionTools:
             
             return f"""Reflection stored successfully.
 ID: {reflection_id}
+Collection: {collection_name}
 Tags: {', '.join(tags) if tags else 'none'}
 Timestamp: {metadata['timestamp']}"""
             
@@ -120,17 +141,34 @@ Timestamp: {metadata['timestamp']}"""
         try:
             # Base path for conversations
             base_path = Path.home() / '.claude' / 'projects'
-            
+
+            # SECURITY FIX: Validate paths to prevent traversal
+            from .security_patches import PathValidator
+            if not PathValidator.is_safe_path(base_path):
+                logger.error(f"Unsafe base path detected: {base_path}")
+                return "<conversation_file><error>Security validation failed</error></conversation_file>"
+
             # If project is specified, try to find it in that project
             if project:
                 # Normalize project name for path matching
-                project_normalized = self.normalize_project_name(project)
-                
+                from .security_patches import InputValidator
+                project_normalized = InputValidator.validate_project_name(
+                    self.normalize_project_name(project)
+                )
+
                 # Look for project directories that match
                 for project_dir in base_path.glob('*'):
+                    # Validate each path before accessing
+                    if not PathValidator.is_safe_path(project_dir):
+                        continue
+
                     if project_normalized in project_dir.name.lower():
                         # Look for JSONL files in this project
                         for jsonl_file in project_dir.glob('*.jsonl'):
+                            # Validate file path
+                            if not PathValidator.is_safe_path(jsonl_file):
+                                continue
+
                             # Check if filename matches conversation_id (with or without .jsonl)
                             if conversation_id in jsonl_file.stem or conversation_id == jsonl_file.stem:
                                 await ctx.debug(f"Found conversation by filename in {jsonl_file}")
@@ -143,8 +181,17 @@ Timestamp: {metadata['timestamp']}"""
             
             # If not found in specific project or no project specified, search all
             await ctx.debug("Searching all projects for conversation")
+            from .security_patches import PathValidator
             for project_dir in base_path.glob('*'):
+                # SECURITY FIX: Validate each path before accessing
+                if not PathValidator.is_safe_path(project_dir):
+                    continue
+
                 for jsonl_file in project_dir.glob('*.jsonl'):
+                    # Validate file path
+                    if not PathValidator.is_safe_path(jsonl_file):
+                        continue
+
                     # Check if filename matches conversation_id (with or without .jsonl)
                     if conversation_id in jsonl_file.stem or conversation_id == jsonl_file.stem:
                         await ctx.debug(f"Found conversation by filename in {jsonl_file}")

package/mcp-server/src/rich_formatting.py

@@ -103,6 +103,109 @@ def format_search_results_rich(
         result_text += f"    <relevance>No conversations matched your query</relevance>\n"
         result_text += f"  </result-summary>\n"
 
+    # Add aggregated insights section (NEW FEATURE)
+    if results and len(results) > 1:
+        result_text += "  <insights>\n"
+        result_text += f"    <!-- Processing {len(results)} results for pattern analysis -->\n"
+
+        # Aggregate file modification patterns
+        file_frequency = {}
+        tool_frequency = {}
+        concept_frequency = {}
+
+        for result in results:
+            # Count file modifications
+            for file in result.get('files_analyzed', []):
+                file_frequency[file] = file_frequency.get(file, 0) + 1
+
+            # Count tool usage
+            for tool in result.get('tools_used', []):
+                tool_frequency[tool] = tool_frequency.get(tool, 0) + 1
+
+            # Count concepts
+            for concept in result.get('concepts', []):
+                concept_frequency[concept] = concept_frequency.get(concept, 0) + 1
+
+        # Show most frequently modified files
+        if file_frequency:
+            top_files = sorted(file_frequency.items(), key=lambda x: x[1], reverse=True)[:3]
+            if top_files:
+                result_text += '    <pattern type="files">\n'
+                result_text += f'      <title>📁 Frequently Modified Files</title>\n'
+                for file, count in top_files:
+                    percentage = (count / len(results)) * 100
+                    result_text += f'      <item count="{count}" pct="{percentage:.0f}%">{file}</item>\n'
+                result_text += '    </pattern>\n'
+
+        # Show common tools used
+        if tool_frequency:
+            top_tools = sorted(tool_frequency.items(), key=lambda x: x[1], reverse=True)[:3]
+            if top_tools:
+                result_text += '    <pattern type="tools">\n'
+                result_text += f'      <title>🔧 Common Tools Used</title>\n'
+                for tool, count in top_tools:
+                    percentage = (count / len(results)) * 100
+                    result_text += f'      <item count="{count}" pct="{percentage:.0f}%">{tool}</item>\n'
+                result_text += '    </pattern>\n'
+
+        # Show related concepts
+        if concept_frequency:
+            top_concepts = sorted(concept_frequency.items(), key=lambda x: x[1], reverse=True)[:3]
+            if top_concepts:
+                result_text += '    <pattern type="concepts">\n'
+                result_text += f'      <title>💡 Related Concepts</title>\n'
+                for concept, count in top_concepts:
+                    percentage = (count / len(results)) * 100
+                    result_text += f'      <item count="{count}" pct="{percentage:.0f}%">{concept}</item>\n'
+                result_text += '    </pattern>\n'
+
+        # Add workflow suggestion based on patterns
+        if file_frequency and tool_frequency:
+            most_common_file = list(file_frequency.keys())[0] if file_frequency else None
+            most_common_tool = list(tool_frequency.keys())[0] if tool_frequency else None
+            if most_common_file and most_common_tool:
+                result_text += '    <suggestion>\n'
+                result_text += f'      <title>💭 Pattern Detection</title>\n'
+                result_text += f'      <text>Similar conversations often involve {most_common_tool} on {most_common_file}</text>\n'
+                result_text += '    </suggestion>\n'
+
+        # Always show a summary even if no clear patterns
+        if not file_frequency and not tool_frequency and not concept_frequency:
+            result_text += '    <summary>\n'
+            result_text += f'      <title>📊 Analysis Summary</title>\n'
+            result_text += f'      <text>Analyzed {len(results)} conversations for patterns</text>\n'
+
+            # Show temporal distribution
+            now = datetime.now(timezone.utc)
+            time_dist = {"today": 0, "week": 0, "month": 0, "older": 0}
+            for result in results:
+                timestamp_str = result.get('timestamp', '')
+                if timestamp_str:
+                    try:
+                        timestamp_clean = timestamp_str.replace('Z', '+00:00') if timestamp_str.endswith('Z') else timestamp_str
+                        timestamp_dt = datetime.fromisoformat(timestamp_clean)
+                        if timestamp_dt.tzinfo is None:
+                            timestamp_dt = timestamp_dt.replace(tzinfo=timezone.utc)
+                        days_ago = (now - timestamp_dt).days
+                        if days_ago == 0:
+                            time_dist["today"] += 1
+                        elif days_ago <= 7:
+                            time_dist["week"] += 1
+                        elif days_ago <= 30:
+                            time_dist["month"] += 1
+                        else:
+                            time_dist["older"] += 1
+                    except:
+                        pass
+
+            if any(time_dist.values()):
+                dist_str = ", ".join([f"{v} {k}" for k, v in time_dist.items() if v > 0])
+                result_text += f'      <temporal>Time distribution: {dist_str}</temporal>\n'
+
+            result_text += '    </summary>\n'
+
+        result_text += "  </insights>\n\n"
+
     # Add metadata
     result_text += f"  <meta>\n"
     result_text += f"    <q>{query}</q>\n"