claude-self-reflect 2.5.4 → 2.5.6

package/installer/setup-wizard-docker.js

@@ -343,13 +343,25 @@ async function importConversations() {
   console.log('\n📚 Checking conversation baseline...');
   
   // Check if baseline exists by looking for imported files state
+  const configDir = path.join(os.homedir(), '.claude-self-reflect', 'config');
   const stateFile = path.join(configDir, 'imported-files.json');
   let hasBaseline = false;
+  let needsMetadataMigration = false;
   
   try {
-    if (fs.existsSync(stateFile)) {
-      const state = JSON.parse(fs.readFileSync(stateFile, 'utf8'));
+    if (fsSync.existsSync(stateFile)) {
+      const state = JSON.parse(fsSync.readFileSync(stateFile, 'utf8'));
       hasBaseline = state.imported_files && Object.keys(state.imported_files).length > 0;
+      
+      // Check if any imported files are in old format (string timestamp vs object)
+      if (hasBaseline) {
+        for (const [file, data] of Object.entries(state.imported_files)) {
+          if (typeof data === 'string') {
+            needsMetadataMigration = true;
+            break;
+          }
+        }
+      }
     }
   } catch (e) {
     // State file doesn't exist or is invalid
@@ -359,13 +371,18 @@ async function importConversations() {
     console.log('\n⚠️  No baseline detected. Initial import STRONGLY recommended.');
     console.log('   Without this, historical conversations won\'t be searchable.');
     console.log('   The watcher only handles NEW conversations going forward.');
+  } else if (needsMetadataMigration) {
+    console.log('\n🔄 Detected old import format. Metadata enhancement available!');
+    console.log('   Re-importing will add file analysis, tool usage, and concept tracking.');
+    console.log('   This enables advanced search features like search_by_file and search_by_concept.');
   }
   
   const answer = await question('\nImport existing Claude conversations? (y/n) [recommended: y]: ');
   
   if (answer.toLowerCase() === 'y') {
-    console.log('🔄 Starting baseline import...');
+    console.log('🔄 Starting baseline import with metadata extraction...');
     console.log('   This ensures ALL your conversations are searchable');
+    console.log('   Enhanced with tool usage tracking and file analysis');
     console.log('   This may take a few minutes depending on your conversation history');
     
     try {
@@ -373,8 +390,9 @@ async function importConversations() {
         cwd: projectRoot,
         stdio: 'inherit'
       });
-      console.log('\n✅ Baseline import completed!');
+      console.log('\n✅ Baseline import completed with metadata!');
       console.log('   Historical conversations are now searchable');
+      console.log('   Tool usage and file analysis metadata extracted');
     } catch {
       console.log('\n⚠️  Import had some issues, but you can continue');
     }

package/mcp-server/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "claude-self-reflect-mcp"
-version = "2.5.1"
+version = "2.5.5"
 description = "MCP server for Claude self-reflection with memory decay"
 # readme = "README.md"
 requires-python = ">=3.10"
@@ -12,7 +12,7 @@ dependencies = [
     "qdrant-client>=1.7.0,<2.0.0",
     "voyageai>=0.1.0,<1.0.0",
     "python-dotenv>=1.0.0,<2.0.0",
-    "pydantic>=2.9.2,<3.0.0",  # Pin to avoid CVE-2024-3772
+    "pydantic>=2.11.7,<3.0.0",  # Updated for fastmcp 2.10.6 compatibility
     "pydantic-settings>=2.0.0,<3.0.0",
     "fastembed>=0.4.0,<1.0.0",
 ]

package/mcp-server/src/server.py

@@ -956,7 +956,14 @@ async def search_by_file(
                 files_analyzed = payload.get('files_analyzed', [])
                 files_edited = payload.get('files_edited', [])
                 
-                if normalized_path in files_analyzed or normalized_path in files_edited:
+                # Check for exact match or if any file ends with the normalized path
+                file_match = False
+                for file in files_analyzed + files_edited:
+                    if file == normalized_path or file.endswith('/' + normalized_path) or file.endswith('\\' + normalized_path):
+                        file_match = True
+                        break
+                
+                if file_match:
                     all_results.append({
                         'score': 1.0,  # File match is always 1.0
                         'payload': payload,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-self-reflect",
-  "version": "2.5.4",
+  "version": "2.5.6",
   "description": "Give Claude perfect memory of all your conversations - Installation wizard for Python MCP server",
   "keywords": [
     "claude",

package/scripts/import-conversations-unified.py

@@ -9,8 +9,9 @@ import json
 import glob
 import hashlib
 import gc
+import re
 from datetime import datetime
-from typing import List, Dict, Any
+from typing import List, Dict, Any, Set
 import logging
 from pathlib import Path
 
@@ -39,11 +40,252 @@ STATE_FILE = os.getenv("STATE_FILE", default_state_file)
 BATCH_SIZE = int(os.getenv("BATCH_SIZE", "10"))  # Reduced from 100 to prevent OOM
 PREFER_LOCAL_EMBEDDINGS = os.getenv("PREFER_LOCAL_EMBEDDINGS", "false").lower() == "true"
 VOYAGE_API_KEY = os.getenv("VOYAGE_KEY")
+CURRENT_METADATA_VERSION = 2  # Version 2: Added tool output extraction
 
 # Set up logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 logger = logging.getLogger(__name__)
 
+# ============= Metadata Extraction Functions =============
+
+def normalize_path_for_metadata(path: str) -> str:
+    """Normalize file paths for consistency in metadata."""
+    if not path:
+        return ""
+    
+    # Remove common prefixes
+    path = path.replace("/Users/", "~/")
+    path = path.replace("\\Users\\", "~\\")
+    
+    # Convert to forward slashes
+    path = path.replace("\\", "/")
+    
+    # Remove duplicate slashes
+    path = re.sub(r'/+', '/', path)
+    
+    return path
+
+def extract_concepts(text: str, tool_usage: Dict[str, Any]) -> Set[str]:
+    """Extract high-level concepts from conversation and tool usage."""
+    concepts = set()
+    
+    # Common development concepts with patterns
+    concept_patterns = {
+        'security': r'(security|vulnerability|CVE|injection|sanitize|escape|auth|token|JWT)',
+        'performance': r'(performance|optimization|speed|memory|efficient|benchmark|latency)',
+        'testing': r'(test|pytest|unittest|coverage|TDD|spec|assert)',
+        'docker': r'(docker|container|compose|dockerfile|kubernetes|k8s)',
+        'api': r'(API|REST|GraphQL|endpoint|webhook|http|request)',
+        'database': r'(database|SQL|query|migration|schema|postgres|mysql|mongodb|qdrant)',
+        'authentication': r'(auth|login|token|JWT|session|oauth|permission)',
+        'debugging': r'(debug|error|exception|traceback|log|stack|trace)',
+        'refactoring': r'(refactor|cleanup|improve|restructure|optimize|technical debt)',
+        'deployment': r'(deploy|CI/CD|release|production|staging|rollout)',
+        'git': r'(git|commit|branch|merge|pull request|PR|rebase)',
+        'architecture': r'(architecture|design|pattern|structure|component|module)',
+        'mcp': r'(MCP|claude-self-reflect|tool|agent|claude code)',
+        'embeddings': r'(embedding|vector|semantic|similarity|fastembed|voyage)',
+        'search': r'(search|query|find|filter|match|relevance)'
+    }
+    
+    # Check text content (limit to first 10000 chars for performance)
+    combined_text = text[:10000].lower() if text else ""
+    for concept, pattern in concept_patterns.items():
+        if re.search(pattern, combined_text, re.IGNORECASE):
+            concepts.add(concept)
+    
+    # Check tool usage patterns
+    if tool_usage.get('grep_searches'):
+        concepts.add('search')
+    if tool_usage.get('files_edited') or tool_usage.get('files_created'):
+        concepts.add('development')
+    if any('test' in str(f).lower() for f in tool_usage.get('files_read', [])):
+        concepts.add('testing')
+    if any('docker' in str(cmd).lower() for cmd in tool_usage.get('bash_commands', [])):
+        concepts.add('docker')
+    
+    return concepts
+
+def extract_files_from_git_output(output_text: str) -> List[str]:
+    """Extract file paths from git command outputs (diff, show, status, etc)."""
+    files = set()
+    
+    # Patterns for different git output formats
+    patterns = [
+        r'diff --git a/(.*?) b/',  # git diff format
+        r'^\+\+\+ b/(.+)$',  # diff new file
+        r'^--- a/(.+)$',  # diff old file
+        r'^modified:\s+(.+)$',  # git status
+        r'^deleted:\s+(.+)$',  # git status
+        r'^new file:\s+(.+)$',  # git status
+        r'^renamed:\s+(.+) -> (.+)$',  # git status (captures both)
+    ]
+    
+    for pattern in patterns:
+        matches = re.findall(pattern, output_text, re.MULTILINE)
+        for match in matches:
+            if isinstance(match, tuple):
+                # Handle renamed files (captures both old and new)
+                for f in match:
+                    if f:
+                        files.add(normalize_path_for_metadata(f))
+            else:
+                files.add(normalize_path_for_metadata(match))
+    
+    return list(files)[:20]  # Limit to 20 files
+
+def extract_tool_data_from_message(tool_use: Dict[str, Any], usage_dict: Dict[str, Any], tool_output: str = None):
+    """Extract tool usage data from a tool_use object in a message, including outputs."""
+    tool_name = tool_use.get('name', '')
+    inputs = tool_use.get('input', {})
+    
+    # Track tool in summary
+    usage_dict['tools_summary'][tool_name] = usage_dict['tools_summary'].get(tool_name, 0) + 1
+    
+    # Handle Read tool
+    if tool_name == 'Read':
+        file_path = inputs.get('file_path')
+        if file_path:
+            normalized = normalize_path_for_metadata(file_path)
+            if normalized not in usage_dict['files_read']:
+                usage_dict['files_read'].append(normalized)
+    
+    # Handle Edit and MultiEdit tools
+    elif tool_name in ['Edit', 'MultiEdit']:
+        path = inputs.get('file_path')
+        if path:
+            normalized = normalize_path_for_metadata(path)
+            if normalized not in usage_dict['files_edited']:
+                usage_dict['files_edited'].append(normalized)
+    
+    # Handle Write tool
+    elif tool_name == 'Write':
+        path = inputs.get('file_path')
+        if path:
+            normalized = normalize_path_for_metadata(path)
+            if normalized not in usage_dict['files_created']:
+                usage_dict['files_created'].append(normalized)
+    
+    # Handle Grep tool
+    elif tool_name == 'Grep':
+        pattern = inputs.get('pattern')
+        if pattern and len(usage_dict['grep_searches']) < 10:  # Limit
+            usage_dict['grep_searches'].append(pattern[:100])  # Truncate long patterns
+    
+    # Handle Bash tool - Extract both command and output
+    elif tool_name == 'Bash':
+        command = inputs.get('command')
+        if command and len(usage_dict['bash_commands']) < 10:
+            usage_dict['bash_commands'].append(command[:200])  # Truncate
+            
+            # Process tool output for git commands
+            if tool_output and any(cmd in command for cmd in ['git diff', 'git show', 'git status']):
+                git_files = extract_files_from_git_output(tool_output)
+                for file_path in git_files:
+                    if file_path not in usage_dict['git_file_changes']:
+                        usage_dict['git_file_changes'].append(file_path)
+    
+    # Store tool output preview (for any tool)
+    if tool_output and len(usage_dict['tool_outputs']) < 15:
+        usage_dict['tool_outputs'].append({
+            'tool': tool_name,
+            'command': inputs.get('command', inputs.get('pattern', ''))[:100],
+            'output_preview': tool_output[:500],  # First 500 chars
+            'output_length': len(tool_output)
+        })
+
+def extract_metadata_from_jsonl(file_path: str) -> Dict[str, Any]:
+    """Extract metadata from a JSONL conversation file."""
+    tool_usage = {
+        "files_read": [],
+        "files_edited": [],
+        "files_created": [],
+        "grep_searches": [],
+        "bash_commands": [],
+        "tools_summary": {},
+        "git_file_changes": [],  # NEW: Files from git outputs
+        "tool_outputs": []  # NEW: Tool output previews
+    }
+    
+    conversation_text = ""
+    tool_outputs = {}  # Map tool_use_id to output text
+    
+    try:
+        # First pass: collect tool outputs
+        with open(file_path, 'r', encoding='utf-8') as f:
+            for line in f:
+                if line.strip():
+                    try:
+                        data = json.loads(line)
+                        if 'message' in data and data['message']:
+                            msg = data['message']
+                            if msg.get('content') and isinstance(msg['content'], list):
+                                for item in msg['content']:
+                                    if isinstance(item, dict) and item.get('type') == 'tool_result':
+                                        # Capture tool output
+                                        tool_id = item.get('tool_use_id')
+                                        output_content = item.get('content', '')
+                                        if tool_id and output_content:
+                                            tool_outputs[tool_id] = output_content
+                        # Also check for toolUseResult in data
+                        if 'toolUseResult' in data:
+                            result = data['toolUseResult']
+                            if isinstance(result, dict):
+                                tool_outputs['last_result'] = json.dumps(result)[:1000]
+                    except:
+                        continue
+        
+        # Second pass: extract tool uses and text with outputs available
+        with open(file_path, 'r', encoding='utf-8') as f:
+            for line in f:
+                if line.strip():
+                    try:
+                        data = json.loads(line)
+                        if 'message' in data and data['message']:
+                            msg = data['message']
+                            # Extract text
+                            if msg.get('content'):
+                                if isinstance(msg['content'], str):
+                                    conversation_text += msg['content'] + "\n"
+                                elif isinstance(msg['content'], list):
+                                    for item in msg['content']:
+                                        if isinstance(item, dict):
+                                            if item.get('type') == 'text' and item.get('text'):
+                                                conversation_text += item['text'] + "\n"
+                                            elif item.get('type') == 'tool_use':
+                                                # Process tool use with output now available
+                                                tool_id = item.get('id', '')
+                                                output = tool_outputs.get(tool_id, '')
+                                                extract_tool_data_from_message(item, tool_usage, output)
+                    except:
+                        continue
+    except Exception as e:
+        logger.warning(f"Error extracting metadata from {file_path}: {e}")
+    
+    # Extract concepts from text
+    concepts = extract_concepts(conversation_text, tool_usage)
+    
+    # Build metadata
+    metadata = {
+        "files_analyzed": tool_usage['files_read'][:20],  # Limit to 20
+        "files_edited": tool_usage['files_edited'][:10],  # Limit to 10
+        "files_created": tool_usage['files_created'][:10],
+        "tools_used": list(tool_usage['tools_summary'].keys())[:20],
+        "tool_summary": dict(list(tool_usage['tools_summary'].items())[:10]),
+        "concepts": list(concepts)[:15],  # Limit to 15
+        "search_patterns": tool_usage['grep_searches'][:10],
+        "git_file_changes": tool_usage['git_file_changes'][:20],  # NEW: Git file changes
+        "tool_outputs": tool_usage['tool_outputs'][:15],  # NEW: Tool output previews
+        "analysis_only": len(tool_usage['files_edited']) == 0 and len(tool_usage['files_created']) == 0,
+        "has_file_metadata": True,
+        "metadata_version": CURRENT_METADATA_VERSION,
+        "metadata_extracted_at": datetime.now().isoformat()
+    }
+    
+    return metadata
+
+# ============= End Metadata Extraction Functions =============
+
 # State management functions
 def load_state():
     """Load the import state from file."""
@@ -273,6 +515,9 @@ def import_project(project_path: Path, collection_name: str, state: dict) -> int
                 created_at = datetime.now().isoformat()
             conversation_id = jsonl_file.stem
             
+            # Extract tool usage metadata from the file
+            metadata = extract_metadata_from_jsonl(str(jsonl_file))
+            
             # Chunk the conversation
             chunks = chunk_conversation(messages)
             
@@ -294,17 +539,22 @@ def import_project(project_path: Path, collection_name: str, state: dict) -> int
                         f"{conversation_id}_{chunk['chunk_index']}".encode()
                     ).hexdigest()[:16]
                     
+                    # Combine basic payload with metadata
+                    payload = {
+                        "text": chunk["text"],
+                        "conversation_id": conversation_id,
+                        "chunk_index": chunk["chunk_index"],
+                        "timestamp": created_at,
+                        "project": project_path.name,
+                        "start_role": chunk["start_role"]
+                    }
+                    # Add metadata fields
+                    payload.update(metadata)
+                    
                     points.append(PointStruct(
                         id=int(point_id, 16) % (2**63),  # Convert to valid integer ID
                         vector=embedding,
-                        payload={
-                            "text": chunk["text"],
-                            "conversation_id": conversation_id,
-                            "chunk_index": chunk["chunk_index"],
-                            "timestamp": created_at,
-                            "project": project_path.name,
-                            "start_role": chunk["start_role"]
-                        }
+                        payload=payload
                     ))
                 
                 # Upload to Qdrant

package/.claude/agents/README.md

@@ -1,155 +0,0 @@
-# Claude Self Reflect - Specialized Agents
-
-This directory contains specialized sub-agents that Claude will proactively use when working on different aspects of the Claude Self Reflect system. Each agent has focused expertise and will automatically activate when their domain is encountered.
-
-## Available Agents
-
-### 🔧 Core System Agents
-
-1. **[qdrant-specialist](./qdrant-specialist.md)** - Vector database expert
-   - Collection management and health monitoring
-   - Search optimization and troubleshooting
-   - Embedding configuration and dimension issues
-   - Performance tuning for Qdrant
-
-2. **[import-debugger](./import-debugger.md)** - Import pipeline specialist
-   - JSONL processing and parsing issues
-   - Conversation chunking optimization
-   - Batch processing and memory management
-   - State tracking and error recovery
-
-3. **[docker-orchestrator](./docker-orchestrator.md)** - Container management expert
-   - Multi-container orchestration
-   - Service health monitoring
-   - Resource optimization
-   - Networking and volume management
-
-4. **[mcp-integration](./mcp-integration.md)** - MCP server developer
-   - Claude Desktop integration
-   - Tool implementation and schemas
-   - TypeScript development
-   - Connection debugging
-
-5. **[search-optimizer](./search-optimizer.md)** - Search quality expert
-   - Semantic search tuning
-   - Embedding model comparison
-   - Similarity threshold optimization
-   - A/B testing methodologies
-
-6. **[reflection-specialist](./reflection-specialist.md)** - Conversation memory expert
-   - Searching past conversations with MCP tools
-   - Storing insights and reflections
-   - Maintaining knowledge continuity
-   - Cross-project conversation search
-
-### 🌟 Open Source Development Agents
-
-7. **[open-source-maintainer](./open-source-maintainer.md)** - Project governance expert
-   - Release management and versioning
-   - Community building and engagement
-   - Issue and PR triage
-   - Contributor recognition
-
-8. **[documentation-writer](./documentation-writer.md)** - Technical documentation specialist
-   - API documentation and references
-   - Tutorial and guide creation
-   - Architecture documentation
-   - Example code development
-
-9. **[performance-tuner](./performance-tuner.md)** - Performance optimization specialist
-   - Search latency optimization
-   - Memory usage reduction
-   - Scalability improvements
-   - Benchmark creation and monitoring
-
-### 🧪 Testing and Validation Agents
-
-10. **[reflect-tester](./reflect-tester.md)** - Comprehensive testing specialist
-   - MCP configuration validation
-   - Tool functionality testing
-   - Collection health verification
-   - Import system validation
-   - Embedding mode testing
-
-## How Agents Work
-
-### Automatic Activation
-
-Claude automatically engages the appropriate agent based on context. For example:
-
-- Mentioning "search returns irrelevant results" → `search-optimizer`
-- Discussing "import showing 0 messages" → `import-debugger`
-- Working on "release v1.2.0" → `open-source-maintainer`
-- Asking about "Qdrant collection errors" → `qdrant-specialist`
-- Requesting "test all reflection functionality" → `reflect-tester`
-- Searching "past conversations about X" → `reflection-specialist`
-
-### Agent Capabilities
-
-Each agent has:
-- **Focused expertise** in their domain
-- **Specific tool permissions** for their tasks
-- **Contextual knowledge** about the project
-- **Best practices** for their area
-
-### Working with Multiple Agents
-
-Agents can collaborate on complex issues:
-
-```
-User: "Search is slow and returning poor results after import"
-→ import-debugger checks data quality
-→ qdrant-specialist optimizes collection settings
-→ search-optimizer tunes similarity thresholds
-→ performance-tuner profiles the entire pipeline
-```
-
-## Creating New Agents
-
-To add a new specialized agent:
-
-1. Create a new `.md` file in this directory
-2. Use the following template:
-
-```markdown
----
-name: agent-name
-description: Brief description for proactive activation
-tools: Read, Write, Edit, Bash, Grep, Glob, LS, WebFetch
----
-
-You are a [role] for the Claude Self Reflect project. Your expertise covers [domains].
-
-## Project Context
-[Specific project knowledge relevant to this agent]
-
-## Key Responsibilities
-[Numbered list of main tasks]
-
-## Essential Commands/Patterns
-[Code blocks with common operations]
-
-## Best Practices
-[Domain-specific guidelines]
-```
-
-3. Update this README with the new agent
-4. Test the agent activation with relevant prompts
-
-## Agent Development Guidelines
-
-- **Be specific**: Agents should have clear, focused roles
-- **Include examples**: Provide code snippets and commands
-- **Stay current**: Update agents as the project evolves
-- **Cross-reference**: Mention when to use other agents
-- **Be helpful**: Include troubleshooting sections
-
-## Maintenance
-
-Agents should be reviewed and updated:
-- When new features are added
-- When common issues emerge
-- When best practices change
-- During major version updates
-
-Remember: These agents are here to help contributors work more effectively on the Claude Self Reflect project. They embody the project's expertise and best practices.