claude-self-reflect 3.3.1 → 4.0.0

package/mcp-server/src/reflection_tools.py

@@ -44,10 +44,24 @@ class ReflectionTools:
         await ctx.debug(f"Storing reflection with {len(tags)} tags")
 
         try:
-            # Determine collection name based on active model type, not prefer_local
+            # Check runtime preference from environment
+            import os
+            prefer_local = os.getenv('PREFER_LOCAL_EMBEDDINGS', 'true').lower() == 'true'
+
             embedding_manager = self.get_embedding_manager()
-            # Use actual model_type to ensure consistency
-            embedding_type = embedding_manager.model_type or ("voyage" if embedding_manager.voyage_client else "local")
+
+            # Use embedding_manager's model_type which already respects preferences
+            embedding_type = embedding_manager.model_type
+
+            if embedding_type == "local":
+                await ctx.debug("Using LOCAL mode (FastEmbed, 384 dimensions)")
+            elif embedding_type == "voyage":
+                await ctx.debug("Using VOYAGE mode (Voyage AI, 1024 dimensions)")
+            else:
+                # Shouldn't happen but handle gracefully
+                embedding_type = "local" if embedding_manager.local_model else "voyage"
+                await ctx.debug(f"Using {embedding_type} mode (fallback)")
+
             collection_name = f"reflections_{embedding_type}"
 
             # Ensure reflections collection exists
@@ -77,8 +91,9 @@ class ReflectionTools:
                 await ctx.debug("Failed to generate embedding for reflection")
                 return "Failed to store reflection: embedding generation failed"
             
-            # Create unique ID
-            reflection_id = hashlib.md5(f"{content}{datetime.now().isoformat()}".encode()).hexdigest()
+            # SECURITY FIX: Use SHA-256 instead of MD5
+            from .security_patches import SecureHashGenerator
+            reflection_id = SecureHashGenerator.generate_id(f"{content}{datetime.now().isoformat()}")
             
             # Prepare metadata
             metadata = {
@@ -104,6 +119,7 @@ class ReflectionTools:
             
             return f"""Reflection stored successfully.
 ID: {reflection_id}
+Collection: {collection_name}
 Tags: {', '.join(tags) if tags else 'none'}
 Timestamp: {metadata['timestamp']}"""
             
@@ -125,17 +141,34 @@ Timestamp: {metadata['timestamp']}"""
         try:
             # Base path for conversations
             base_path = Path.home() / '.claude' / 'projects'
-            
+
+            # SECURITY FIX: Validate paths to prevent traversal
+            from .security_patches import PathValidator
+            if not PathValidator.is_safe_path(base_path):
+                logger.error(f"Unsafe base path detected: {base_path}")
+                return "<conversation_file><error>Security validation failed</error></conversation_file>"
+
             # If project is specified, try to find it in that project
             if project:
                 # Normalize project name for path matching
-                project_normalized = self.normalize_project_name(project)
-                
+                from .security_patches import InputValidator
+                project_normalized = InputValidator.validate_project_name(
+                    self.normalize_project_name(project)
+                )
+
                 # Look for project directories that match
                 for project_dir in base_path.glob('*'):
+                    # Validate each path before accessing
+                    if not PathValidator.is_safe_path(project_dir):
+                        continue
+
                     if project_normalized in project_dir.name.lower():
                         # Look for JSONL files in this project
                         for jsonl_file in project_dir.glob('*.jsonl'):
+                            # Validate file path
+                            if not PathValidator.is_safe_path(jsonl_file):
+                                continue
+
                             # Check if filename matches conversation_id (with or without .jsonl)
                             if conversation_id in jsonl_file.stem or conversation_id == jsonl_file.stem:
                                 await ctx.debug(f"Found conversation by filename in {jsonl_file}")
@@ -148,8 +181,17 @@ Timestamp: {metadata['timestamp']}"""
             
             # If not found in specific project or no project specified, search all
             await ctx.debug("Searching all projects for conversation")
+            from .security_patches import PathValidator
             for project_dir in base_path.glob('*'):
+                # SECURITY FIX: Validate each path before accessing
+                if not PathValidator.is_safe_path(project_dir):
+                    continue
+
                 for jsonl_file in project_dir.glob('*.jsonl'):
+                    # Validate file path
+                    if not PathValidator.is_safe_path(jsonl_file):
+                        continue
+
                     # Check if filename matches conversation_id (with or without .jsonl)
                     if conversation_id in jsonl_file.stem or conversation_id == jsonl_file.stem:
                         await ctx.debug(f"Found conversation by filename in {jsonl_file}")

package/mcp-server/src/rich_formatting.py

@@ -103,6 +103,109 @@ def format_search_results_rich(
         result_text += f"    <relevance>No conversations matched your query</relevance>\n"
         result_text += f"  </result-summary>\n"
 
+    # Add aggregated insights section (NEW FEATURE)
+    if results and len(results) > 1:
+        result_text += "  <insights>\n"
+        result_text += f"    <!-- Processing {len(results)} results for pattern analysis -->\n"
+
+        # Aggregate file modification patterns
+        file_frequency = {}
+        tool_frequency = {}
+        concept_frequency = {}
+
+        for result in results:
+            # Count file modifications
+            for file in result.get('files_analyzed', []):
+                file_frequency[file] = file_frequency.get(file, 0) + 1
+
+            # Count tool usage
+            for tool in result.get('tools_used', []):
+                tool_frequency[tool] = tool_frequency.get(tool, 0) + 1
+
+            # Count concepts
+            for concept in result.get('concepts', []):
+                concept_frequency[concept] = concept_frequency.get(concept, 0) + 1
+
+        # Show most frequently modified files
+        if file_frequency:
+            top_files = sorted(file_frequency.items(), key=lambda x: x[1], reverse=True)[:3]
+            if top_files:
+                result_text += '    <pattern type="files">\n'
+                result_text += f'      <title>📁 Frequently Modified Files</title>\n'
+                for file, count in top_files:
+                    percentage = (count / len(results)) * 100
+                    result_text += f'      <item count="{count}" pct="{percentage:.0f}%">{file}</item>\n'
+                result_text += '    </pattern>\n'
+
+        # Show common tools used
+        if tool_frequency:
+            top_tools = sorted(tool_frequency.items(), key=lambda x: x[1], reverse=True)[:3]
+            if top_tools:
+                result_text += '    <pattern type="tools">\n'
+                result_text += f'      <title>🔧 Common Tools Used</title>\n'
+                for tool, count in top_tools:
+                    percentage = (count / len(results)) * 100
+                    result_text += f'      <item count="{count}" pct="{percentage:.0f}%">{tool}</item>\n'
+                result_text += '    </pattern>\n'
+
+        # Show related concepts
+        if concept_frequency:
+            top_concepts = sorted(concept_frequency.items(), key=lambda x: x[1], reverse=True)[:3]
+            if top_concepts:
+                result_text += '    <pattern type="concepts">\n'
+                result_text += f'      <title>💡 Related Concepts</title>\n'
+                for concept, count in top_concepts:
+                    percentage = (count / len(results)) * 100
+                    result_text += f'      <item count="{count}" pct="{percentage:.0f}%">{concept}</item>\n'
+                result_text += '    </pattern>\n'
+
+        # Add workflow suggestion based on patterns
+        if file_frequency and tool_frequency:
+            most_common_file = list(file_frequency.keys())[0] if file_frequency else None
+            most_common_tool = list(tool_frequency.keys())[0] if tool_frequency else None
+            if most_common_file and most_common_tool:
+                result_text += '    <suggestion>\n'
+                result_text += f'      <title>💭 Pattern Detection</title>\n'
+                result_text += f'      <text>Similar conversations often involve {most_common_tool} on {most_common_file}</text>\n'
+                result_text += '    </suggestion>\n'
+
+        # Always show a summary even if no clear patterns
+        if not file_frequency and not tool_frequency and not concept_frequency:
+            result_text += '    <summary>\n'
+            result_text += f'      <title>📊 Analysis Summary</title>\n'
+            result_text += f'      <text>Analyzed {len(results)} conversations for patterns</text>\n'
+
+            # Show temporal distribution
+            now = datetime.now(timezone.utc)
+            time_dist = {"today": 0, "week": 0, "month": 0, "older": 0}
+            for result in results:
+                timestamp_str = result.get('timestamp', '')
+                if timestamp_str:
+                    try:
+                        timestamp_clean = timestamp_str.replace('Z', '+00:00') if timestamp_str.endswith('Z') else timestamp_str
+                        timestamp_dt = datetime.fromisoformat(timestamp_clean)
+                        if timestamp_dt.tzinfo is None:
+                            timestamp_dt = timestamp_dt.replace(tzinfo=timezone.utc)
+                        days_ago = (now - timestamp_dt).days
+                        if days_ago == 0:
+                            time_dist["today"] += 1
+                        elif days_ago <= 7:
+                            time_dist["week"] += 1
+                        elif days_ago <= 30:
+                            time_dist["month"] += 1
+                        else:
+                            time_dist["older"] += 1
+                    except:
+                        pass
+
+            if any(time_dist.values()):
+                dist_str = ", ".join([f"{v} {k}" for k, v in time_dist.items() if v > 0])
+                result_text += f'      <temporal>Time distribution: {dist_str}</temporal>\n'
+
+            result_text += '    </summary>\n'
+
+        result_text += "  </insights>\n\n"
+
     # Add metadata
     result_text += f"  <meta>\n"
     result_text += f"    <q>{query}</q>\n"

package/mcp-server/src/search_tools.py

@@ -83,10 +83,20 @@ class SearchTools:
             # Generate embedding for query
             embedding_manager = self.get_embedding_manager()
             
-            # Determine embedding type based on collection name
-            embedding_type = 'voyage' if collection_name.endswith('_voyage') else 'local'
+            # Determine embedding type based on collection name (v3 and v4 compatible)
+            # v4 format: csr_project_mode_dims (e.g., csr_project_cloud_1024d)
+            # v3 format: project_suffix (e.g., project_voyage)
+            if '_cloud_' in collection_name or collection_name.endswith('_1024d') or collection_name.endswith('_voyage'):
+                embedding_type = 'voyage'
+            else:
+                embedding_type = 'local'
             query_embedding = await embedding_manager.generate_embedding(query, force_type=embedding_type)
-            
+
+            # FIX: Validate embedding before search
+            if query_embedding is None:
+                logger.warning(f"Embedding generation failed for query in {collection_name}")
+                return []
+
             # Search the collection
             search_results = await self.qdrant_client.search(
                 collection_name=collection_name,
@@ -132,9 +142,9 @@ class SearchTools:
                     # Apply exponential decay
                     decay_factor = pow(2, -age / self.decay_scale_days)
                     
-                    # Adjust score
+                    # Adjust score - FIX: Maintain comparable scale
                     original_score = result['score']
-                    result['score'] = original_score * (1 - self.decay_weight) + decay_factor * self.decay_weight
+                    result['score'] = original_score * ((1 - self.decay_weight) + self.decay_weight * decay_factor)
                     result['original_score'] = original_score
                     result['decay_factor'] = decay_factor
                     
@@ -242,12 +252,14 @@ class SearchTools:
                 ]
                 await ctx.debug(f"Filtered to {len(filtered_collections)} collections from {len(all_collections)} total")
             else:
-                # Use all collections except reflections
+                # Use all collections INCLUDING reflections (with decay)
                 collections_response = await self.qdrant_client.get_collections()
                 collections = collections_response.collections
+                # Include both conversation collections and reflection collections
                 filtered_collections = [
-                    c for c in collections 
-                    if not c.name.startswith('reflections')
+                    c for c in collections
+                    if (c.name.endswith('_local') or c.name.endswith('_voyage') or
+                        c.name.startswith('reflections'))
                 ]
                 await ctx.debug(f"Searching across {len(filtered_collections)} collections")
             
@@ -362,12 +374,14 @@ class SearchTools:
                     if c.name in collection_names
                 ]
             else:
-                # Use all collections except reflections
+                # Use all collections INCLUDING reflections (with decay)
                 collections_response = await self.qdrant_client.get_collections()
                 collections = collections_response.collections
+                # Include both conversation collections and reflection collections
                 filtered_collections = [
-                    c for c in collections 
-                    if not c.name.startswith('reflections')
+                    c for c in collections
+                    if (c.name.endswith('_local') or c.name.endswith('_voyage') or
+                        c.name.startswith('reflections'))
                 ]
             
             # Quick PARALLEL count across collections
@@ -450,12 +464,14 @@ class SearchTools:
                     if c.name in collection_names
                 ]
             else:
-                # Use all collections except reflections
+                # Use all collections INCLUDING reflections (with decay)
                 collections_response = await self.qdrant_client.get_collections()
                 collections = collections_response.collections
+                # Include both conversation collections and reflection collections
                 filtered_collections = [
-                    c for c in collections 
-                    if not c.name.startswith('reflections')
+                    c for c in collections
+                    if (c.name.endswith('_local') or c.name.endswith('_voyage') or
+                        c.name.startswith('reflections'))
                 ]
             
             # Gather results for summary using PARALLEL search
@@ -545,12 +561,14 @@ class SearchTools:
                     if c.name in collection_names
                 ]
             else:
-                # Use all collections except reflections
+                # Use all collections INCLUDING reflections (with decay)
                 collections_response = await self.qdrant_client.get_collections()
                 collections = collections_response.collections
+                # Include both conversation collections and reflection collections
                 filtered_collections = [
-                    c for c in collections 
-                    if not c.name.startswith('reflections')
+                    c for c in collections
+                    if (c.name.endswith('_local') or c.name.endswith('_voyage') or
+                        c.name.startswith('reflections'))
                 ]
             
             # Gather all results using PARALLEL search
@@ -698,17 +716,21 @@ class SearchTools:
                     await ctx.debug(f"Error searching {collection_name}: {e}")
                     return []
             
-            # Use asyncio.gather for PARALLEL search across all collections
+            # SECURITY FIX: Use proper concurrency limiting
             import asyncio
+            from .security_patches import ConcurrencyLimiter
+
             search_tasks = [search_collection(c.name) for c in collections]
-            
-            # Limit concurrent searches to avoid overload
-            batch_size = 20
+
+            # Use semaphore-based limiting instead of batching
             all_results = []
-            for i in range(0, len(search_tasks), batch_size):
-                batch = search_tasks[i:i+batch_size]
-                batch_results = await asyncio.gather(*batch)
-                for results in batch_results:
+            batch_results = await ConcurrencyLimiter.limited_gather(search_tasks, limit=10)
+            for results in batch_results:
+                if isinstance(results, Exception):
+                    logger.error(f"Search task failed: {type(results).__name__}: {results}")
+                    await ctx.debug(f"Search task error: {results}")
+                    continue
+                if results:
                     all_results.extend(results)
             
             # Format results
@@ -791,7 +813,7 @@ def register_search_tools(
         project_resolver  # Pass the resolver
     )
     
-    @mcp.tool()
+    @mcp.tool(name="csr_reflect_on_past")
     async def reflect_on_past(
         ctx: Context,
         query: str = Field(description="The search query to find semantically similar conversations"),
@@ -804,29 +826,45 @@ def register_search_tools(
         include_raw: bool = Field(default=False, description="Include raw Qdrant payload data for debugging (increases response size)"),
         response_format: str = Field(default="xml", description="Response format: 'xml' or 'markdown'")
     ) -> str:
-        """Search for relevant past conversations using semantic search with optional time decay."""
+        """Search past Claude conversations semantically to find relevant context.
+
+        WHEN TO USE: User asks 'what did we discuss about X?', 'find conversations about Y',
+        mentions 'remember when' or 'last time', debugging issues that may have been solved before,
+        or finding implementation patterns used in the project.
+
+        This is the PRIMARY tool for conversation memory - use it liberally!"""
         return await tools.reflect_on_past(ctx, query, limit, min_score, use_decay, project, mode, brief, include_raw, response_format)
     
-    @mcp.tool()
+    @mcp.tool(name="csr_quick_check")
     async def quick_search(
         ctx: Context,
         query: str = Field(description="The search query to find semantically similar conversations"),
         min_score: float = Field(default=0.3, description="Minimum similarity score (0-1)"),
         project: Optional[str] = Field(default=None, description="Search specific project only. If not provided, searches current project based on working directory. Use 'all' to search across all projects.")
     ) -> str:
-        """Quick search that returns only the count and top result for fast overview."""
+        """Quick check if a topic was discussed before (returns count + top match only).
+
+        WHEN TO USE: User asks 'have we discussed X?' or 'is there anything about Y?',
+        need a yes/no answer about topic existence, checking if a problem was encountered before.
+
+        Much faster than full search - use for existence checks!"""
         return await tools.quick_search(ctx, query, min_score, project)
     
-    @mcp.tool()
+    @mcp.tool(name="csr_search_insights")
     async def search_summary(
         ctx: Context,
         query: str = Field(description="The search query to find semantically similar conversations"),
         project: Optional[str] = Field(default=None, description="Search specific project only. If not provided, searches current project based on working directory. Use 'all' to search across all projects.")
     ) -> str:
-        """Get aggregated insights from search results without individual result details."""
+        """Get aggregated insights and patterns from search results.
+
+        WHEN TO USE: User wants patterns or trends, analyzing topic evolution,
+        understanding common themes, getting high-level view without details.
+
+        Provides analysis, not just search results!"""
         return await tools.search_summary(ctx, query, project)
     
-    @mcp.tool()
+    @mcp.tool(name="csr_get_more")
     async def get_more_results(
         ctx: Context,
         query: str = Field(description="The original search query"),
@@ -835,20 +873,30 @@ def register_search_tools(
         min_score: float = Field(default=0.3, description="Minimum similarity score (0-1)"),
         project: Optional[str] = Field(default=None, description="Search specific project only")
     ) -> str:
-        """Get additional search results after an initial search (pagination support)."""
+        """Get additional search results for paginated exploration.
+
+        WHEN TO USE: User says 'show me more' after a search, initial results weren't sufficient,
+        deep diving into a topic, user wants comprehensive coverage.
+
+        Use after initial search when more context is needed!"""
         return await tools.get_more_results(ctx, query, offset, limit, min_score, project)
     
-    @mcp.tool()
+    @mcp.tool(name="csr_search_by_file")
     async def search_by_file(
         ctx: Context,
         file_path: str = Field(description="The file path to search for in conversations"),
         limit: int = Field(default=10, description="Maximum number of results to return"),
         project: Optional[str] = Field(default=None, description="Search specific project only. Use 'all' to search across all projects.")
     ) -> str:
-        """Search for conversations that analyzed a specific file."""
+        """Find all conversations that analyzed or modified a specific file.
+
+        WHEN TO USE: User asks 'when did we modify X file?', investigating file history,
+        understanding why changes were made, finding discussions about specific code files.
+
+        Perfect for code archaeology and understanding file evolution!"""
         return await tools.search_by_file(ctx, file_path, limit, project)
     
-    @mcp.tool()
+    @mcp.tool(name="csr_search_by_concept")
     async def search_by_concept(
         ctx: Context,
         concept: str = Field(description="The concept to search for (e.g., 'security', 'docker', 'testing')"),
@@ -856,7 +904,12 @@ def register_search_tools(
         project: Optional[str] = Field(default=None, description="Search specific project only. Use 'all' to search across all projects."),
         include_files: bool = Field(default=True, description="Include file information in results")
     ) -> str:
-        """Search for conversations about a specific development concept."""
+        """Search for conversations about specific development concepts or themes.
+
+        WHEN TO USE: User asks about broad topics like 'security', 'testing', 'performance',
+        looking for all discussions on a technical theme, gathering knowledge about a concept.
+
+        Ideal for thematic analysis and knowledge gathering!"""
         return await tools.search_by_concept(ctx, concept, limit, project, include_files)
     
     @mcp.tool()