claude-self-reflect 2.3.6 → 2.3.8

package/README.md

@@ -4,7 +4,7 @@ Claude forgets everything. This fixes that.
 
 ## What You Get
 
-Ask Claude about past conversations. Get actual answers. Local-first with no cloud dependencies, but cloud-enhanced search available when you need it.
+Ask Claude about past conversations. Get actual answers. **100% local by default** - your conversations never leave your machine. Cloud-enhanced search available when you need it.
 
 **Before**: "I don't have access to previous conversations"  
 **After**: 
@@ -143,6 +143,21 @@ Built by developers tired of re-explaining context every conversation.
 - Python 3.10+
 - 5 minutes for setup
 
+## Upgrading from Earlier Versions
+
+**v2.3.7+ includes major improvements:**
+- **Privacy First**: Local embeddings by default - your data never leaves your machine
+- **Smarter Setup**: Handles existing installations gracefully
+- **Better Security**: Automated vulnerability scanning
+
+**To upgrade:**
+```bash
+npm update -g claude-self-reflect
+claude-self-reflect setup  # Re-run setup, it handles everything
+```
+
+The setup wizard now detects and fixes common upgrade issues automatically. Your existing conversations remain searchable.
+
 ## Advanced Setup
 
 Want to customize? See [Configuration Guide](docs/installation-guide.md).

package/installer/setup-wizard.js

@@ -245,15 +245,48 @@ async function setupPythonEnvironment() {
     // Check if venv already exists
     const venvPath = join(mcpPath, 'venv');
     let venvExists = false;
+    let venvHealthy = false;
+    let needsInstall = false;
+    
     try {
       await fs.access(venvPath);
       venvExists = true;
-      console.log('✅ Virtual environment already exists');
+      
+      // Check if existing venv is healthy
+      const venvPython = process.platform === 'win32'
+        ? join(venvPath, 'Scripts', 'python.exe')
+        : join(venvPath, 'bin', 'python');
+      
+      try {
+        // Try to run python --version to verify venv is functional
+        safeExec(venvPython, ['--version'], { stdio: 'pipe' });
+        
+        // Also check if MCP dependencies are installed
+        try {
+          safeExec(venvPython, ['-c', 'import fastmcp, qdrant_client'], { stdio: 'pipe' });
+          venvHealthy = true;
+          console.log('✅ Virtual environment already exists and is healthy');
+        } catch {
+          // Dependencies not installed, need to install them
+          console.log('⚠️  Virtual environment exists but missing dependencies');
+          needsInstall = true;
+          venvHealthy = true; // venv itself is healthy, just needs deps
+        }
+      } catch (healthError) {
+        console.log('⚠️  Existing virtual environment is corrupted');
+        console.log('   Removing and recreating...');
+        
+        // Remove broken venv
+        const { rmSync } = await import('fs');
+        rmSync(venvPath, { recursive: true, force: true });
+        venvExists = false;
+        venvHealthy = false;
+      }
     } catch {
       // venv doesn't exist, create it
     }
     
-    if (!venvExists) {
+    if (!venvExists || !venvHealthy) {
       // Create virtual environment
       console.log('Creating virtual environment...');
       const pythonCmd = process.env.PYTHON_PATH || 'python3';
@@ -265,6 +298,7 @@ async function setupPythonEnvironment() {
           stdio: 'inherit'
         });
         if (result.error) throw result.error;
+        needsInstall = true; // Mark that we need to install dependencies
       } catch (venvError) {
         console.log('⚠️  Failed to create venv with python3, trying python...');
         try {
@@ -274,6 +308,7 @@ async function setupPythonEnvironment() {
             stdio: 'inherit'
           });
           if (result.error) throw result.error;
+          needsInstall = true; // Mark that we need to install dependencies
         } catch {
           console.log('❌ Failed to create virtual environment');
           console.log('📚 Fix: Install python3-venv package');
@@ -285,7 +320,6 @@ async function setupPythonEnvironment() {
     }
     
     // Setup paths for virtual environment
-    console.log('Setting up pip in virtual environment...');
     const venvPython = process.platform === 'win32'
       ? join(mcpPath, 'venv', 'Scripts', 'python.exe')
       : join(mcpPath, 'venv', 'bin', 'python');
@@ -293,15 +327,19 @@ async function setupPythonEnvironment() {
       ? join(mcpPath, 'venv', 'Scripts', 'pip.exe')
       : join(mcpPath, 'venv', 'bin', 'pip');
     
-    // First, try to install certifi to help with SSL issues
-    console.log('Installing certificate handler...');
-    try {
-      safeExec(venvPip, [
-        'install', '--trusted-host', 'pypi.org', '--trusted-host', 'files.pythonhosted.org', 'certifi'
-      ], { cwd: mcpPath, stdio: 'pipe' });
-    } catch {
-      // Continue even if certifi fails
-    }
+    // Only install dependencies if we just created a new venv
+    if (needsInstall) {
+      console.log('Setting up pip in virtual environment...');
+      
+      // First, try to install certifi to help with SSL issues
+      console.log('Installing certificate handler...');
+      try {
+        safeExec(venvPip, [
+          'install', '--trusted-host', 'pypi.org', '--trusted-host', 'files.pythonhosted.org', 'certifi'
+        ], { cwd: mcpPath, stdio: 'pipe' });
+      } catch {
+        // Continue even if certifi fails
+      }
     
     // Upgrade pip and install wheel first
     try {
@@ -369,9 +407,9 @@ async function setupPythonEnvironment() {
       }
     }
     
-    // Install script dependencies
-    console.log('Installing import script dependencies...');
-    try {
+      // Install script dependencies
+      console.log('Installing import script dependencies...');
+      try {
       safeExec(venvPip, [
         'install', '-r', join(scriptsPath, 'requirements.txt')
       ], { cwd: mcpPath, stdio: 'inherit' });
@@ -387,6 +425,7 @@ async function setupPythonEnvironment() {
         console.log('   You may need to install them manually later');
       }
     }
+    } // End of needsInstall block
     
     console.log('✅ Python environment setup complete');
     return true;
@@ -679,7 +718,7 @@ async function importConversations() {
   
   try {
     const pythonCmd = process.env.PYTHON_PATH || 'python3';
-    const importScript = join(projectRoot, 'scripts', 'import-conversations-voyage.py');
+    const importScript = join(projectRoot, 'scripts', 'import-conversations-unified.py');
     
     // Use the venv Python directly - platform specific
     let venvPython;
@@ -961,12 +1000,13 @@ async function showSystemDashboard() {
 async function setupWatcher() {
   console.log('\n⚙️  Setting up Continuous Import (Watcher)...');
   
-  const watcherScript = join(projectRoot, 'scripts', 'import-watcher.py');
+  // Check if Docker compose file exists
+  const dockerComposeFile = join(projectRoot, 'docker-compose.yaml');
   
-  // Check if watcher exists
+  // Check if Docker watcher is available
   try {
-    await fs.access(watcherScript);
-    console.log('✅ Watcher script found');
+    await fs.access(dockerComposeFile);
+    console.log('✅ Docker-based watcher available');
     
     // Watcher works with both local and cloud embeddings
     console.log(localMode ? '🏠 Watcher will use local embeddings' : '🌐 Watcher will use Voyage AI embeddings');
@@ -1079,15 +1119,15 @@ async function setupWatcher() {
         // Fallback to manual Python execution
         console.log('\n📝 To enable the watcher, run:');
         console.log('   cd claude-self-reflect');
-        console.log('   source mcp-server/venv/bin/activate');
-        console.log('   python scripts/import-watcher.py &');
+        console.log('   docker compose --profile watch up -d');
       }
     } else {
       console.log('\n📝 You can enable the watcher later by running:');
       console.log('   docker compose --profile watch up -d');
     }
   } catch {
-    console.log('⚠️  Watcher script not found');
+    console.log('⚠️  Docker compose not found');
+    console.log('   The watcher requires Docker to run continuously');
   }
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-self-reflect",
-  "version": "2.3.6",
+  "version": "2.3.8",
   "description": "Give Claude perfect memory of all your conversations - Installation wizard for Python MCP server",
   "keywords": [
     "claude",
@@ -47,4 +47,4 @@
     "access": "public",
     "registry": "https://registry.npmjs.org/"
   }
-}
+}

package/scripts/import-conversations-isolated.py

@@ -1,311 +0,0 @@
-#!/usr/bin/env python3
-"""
-Import Claude conversation logs with project isolation support.
-Each project gets its own collection for complete isolation.
-"""
-
-import json
-import os
-import glob
-import hashlib
-from datetime import datetime, timedelta
-from typing import List, Dict, Any, Set
-import logging
-from qdrant_client import QdrantClient
-from qdrant_client.models import (
-    VectorParams, Distance, PointStruct,
-    Filter, FieldCondition, MatchValue
-)
-from sentence_transformers import SentenceTransformer
-
-# Configuration
-QDRANT_URL = os.getenv("QDRANT_URL", "http://localhost:6333")
-LOGS_DIR = os.getenv("LOGS_DIR", "/logs")
-STATE_FILE = os.getenv("STATE_FILE", "/config/imported-files.json")
-EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL", "sentence-transformers/all-MiniLM-L6-v2")
-BATCH_SIZE = int(os.getenv("BATCH_SIZE", "100"))
-ISOLATION_MODE = os.getenv("ISOLATION_MODE", "isolated")  # isolated, shared, hybrid
-
-# Set up logging
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
-logger = logging.getLogger(__name__)
-
-class ProjectAwareImporter:
-    def __init__(self):
-        """Initialize the importer with Qdrant client and embedding model."""
-        self.client = QdrantClient(url=QDRANT_URL)
-        self.encoder = SentenceTransformer(EMBEDDING_MODEL)
-        self.imported_files = self.load_state()
-        self.project_collections: Set[str] = set()
-        
-    def load_state(self) -> Dict[str, Set[str]]:
-        """Load the set of already imported files per project."""
-        if os.path.exists(STATE_FILE):
-            try:
-                with open(STATE_FILE, 'r') as f:
-                    data = json.load(f)
-                    # Convert to per-project tracking
-                    if isinstance(data.get('files'), list):
-                        # Legacy format - convert to new format
-                        return {'_legacy': set(data['files'])}
-                    else:
-                        # New format with per-project tracking
-                        return {k: set(v) for k, v in data.get('projects', {}).items()}
-            except Exception as e:
-                logger.error(f"Failed to load state: {e}")
-        return {}
-    
-    def save_state(self):
-        """Save the set of imported files per project."""
-        os.makedirs(os.path.dirname(STATE_FILE), exist_ok=True)
-        with open(STATE_FILE, 'w') as f:
-            json.dump({
-                'projects': {k: list(v) for k, v in self.imported_files.items()},
-                'last_updated': datetime.now().isoformat(),
-                'mode': ISOLATION_MODE
-            }, f, indent=2)
-    
-    def get_collection_name(self, project_name: str) -> str:
-        """Get collection name based on isolation mode."""
-        if ISOLATION_MODE == "isolated":
-            # Create project-specific collection name
-            project_hash = hashlib.md5(project_name.encode()).hexdigest()[:8]
-            return f"conv_{project_hash}"
-        else:
-            # Shared collection mode
-            return "conversations"
-    
-    def setup_collection(self, project_name: str):
-        """Create or update the Qdrant collection for a project."""
-        collection_name = self.get_collection_name(project_name)
-        
-        # Skip if already set up in this session
-        if collection_name in self.project_collections:
-            return collection_name
-        
-        collections = self.client.get_collections().collections
-        exists = any(c.name == collection_name for c in collections)
-        
-        if not exists:
-            logger.info(f"Creating collection: {collection_name} for project: {project_name}")
-            self.client.create_collection(
-                collection_name=collection_name,
-                vectors_config=VectorParams(
-                    size=384,  # all-MiniLM-L6-v2 dimension
-                    distance=Distance.COSINE
-                )
-            )
-        else:
-            logger.info(f"Collection {collection_name} already exists for project: {project_name}")
-        
-        self.project_collections.add(collection_name)
-        return collection_name
-    
-    def extract_project_name(self, file_path: str) -> str:
-        """Extract project name from file path."""
-        # Expected path: /logs/<project-name>/<conversation-id>.jsonl
-        parts = file_path.split('/')
-        if len(parts) >= 3 and parts[-2] != 'logs':
-            return parts[-2]
-        return 'unknown'
-    
-    def process_jsonl_file(self, file_path: str) -> List[Dict[str, Any]]:
-        """Extract messages from a JSONL file."""
-        messages = []
-        
-        try:
-            with open(file_path, 'r') as f:
-                for line_num, line in enumerate(f, 1):
-                    try:
-                        data = json.loads(line.strip())
-                        
-                        # Extract message if present
-                        if 'message' in data and data['message']:
-                            msg = data['message']
-                            if 'role' in msg and 'content' in msg:
-                                # Handle content that might be an object
-                                content = msg['content']
-                                if isinstance(content, dict):
-                                    content = content.get('text', json.dumps(content))
-                                
-                                # Create message document
-                                messages.append({
-                                    'role': msg['role'],
-                                    'content': content,
-                                    'file_path': file_path,
-                                    'line_number': line_num,
-                                    'timestamp': data.get('timestamp', datetime.now().isoformat())
-                                })
-                    except json.JSONDecodeError:
-                        logger.warning(f"Failed to parse line {line_num} in {file_path}")
-                    except Exception as e:
-                        logger.error(f"Error processing line {line_num} in {file_path}: {e}")
-                        
-        except Exception as e:
-            logger.error(f"Failed to read file {file_path}: {e}")
-        
-        return messages
-    
-    def create_conversation_chunks(self, messages: List[Dict[str, Any]], chunk_size: int = 5) -> List[Dict[str, Any]]:
-        """Group messages into conversation chunks for better context."""
-        chunks = []
-        
-        for i in range(0, len(messages), chunk_size):
-            chunk_messages = messages[i:i + chunk_size]
-            
-            # Create a conversation summary
-            conversation_text = "\n\n".join([
-                f"{msg['role'].upper()}: {msg['content'][:500]}..."
-                if len(msg['content']) > 500 else f"{msg['role'].upper()}: {msg['content']}"
-                for msg in chunk_messages
-            ])
-            
-            # Extract metadata
-            project_id = self.extract_project_name(chunk_messages[0]['file_path'])
-            conversation_id = os.path.basename(chunk_messages[0]['file_path']).replace('.jsonl', '')
-            
-            chunks.append({
-                'id': hashlib.md5(f"{chunk_messages[0]['file_path']}_{i}".encode()).hexdigest(),
-                'text': conversation_text,
-                'metadata': {
-                    'project_id': project_id,
-                    'project_name': project_id,  # Add both for compatibility
-                    'conversation_id': conversation_id,
-                    'chunk_index': i // chunk_size,
-                    'message_count': len(chunk_messages),
-                    'start_role': chunk_messages[0]['role'],
-                    'timestamp': chunk_messages[0]['timestamp'],
-                    'file_path': chunk_messages[0]['file_path']
-                }
-            })
-        
-        return chunks
-    
-    def import_to_qdrant(self, chunks: List[Dict[str, Any]], collection_name: str):
-        """Import conversation chunks to a specific Qdrant collection."""
-        if not chunks:
-            return
-        
-        # Generate embeddings
-        texts = [chunk['text'] for chunk in chunks]
-        embeddings = self.encoder.encode(texts, show_progress_bar=True)
-        
-        # Create points for Qdrant
-        points = []
-        for i, (chunk, embedding) in enumerate(zip(chunks, embeddings)):
-            points.append(
-                PointStruct(
-                    id=chunk['id'],
-                    vector=embedding.tolist(),
-                    payload={
-                        'text': chunk['text'],
-                        **chunk['metadata']
-                    }
-                )
-            )
-        
-        # Upload to Qdrant in batches
-        for i in range(0, len(points), BATCH_SIZE):
-            batch = points[i:i + BATCH_SIZE]
-            self.client.upsert(
-                collection_name=collection_name,
-                points=batch
-            )
-            logger.info(f"Uploaded batch of {len(batch)} points to {collection_name}")
-    
-    def find_recent_files(self, days: int = 30) -> List[str]:
-        """Find JSONL files modified in the last N days."""
-        cutoff_time = datetime.now() - timedelta(days=days)
-        pattern = os.path.join(LOGS_DIR, "**", "*.jsonl")
-        
-        recent_files = []
-        for file_path in glob.glob(pattern, recursive=True):
-            try:
-                mtime = os.path.getmtime(file_path)
-                if datetime.fromtimestamp(mtime) >= cutoff_time:
-                    recent_files.append(file_path)
-            except Exception as e:
-                logger.error(f"Error checking file {file_path}: {e}")
-        
-        return recent_files
-    
-    def run(self):
-        """Main import process with project isolation."""
-        logger.info(f"Starting conversation import to Qdrant (mode: {ISOLATION_MODE})")
-        
-        # Find files to import
-        all_files = self.find_recent_files()
-        logger.info(f"Found {len(all_files)} total files")
-        
-        # Group files by project
-        files_by_project: Dict[str, List[str]] = {}
-        for file_path in all_files:
-            project_name = self.extract_project_name(file_path)
-            if project_name not in files_by_project:
-                files_by_project[project_name] = []
-            files_by_project[project_name].append(file_path)
-        
-        logger.info(f"Found {len(files_by_project)} projects to process")
-        
-        total_chunks = 0
-        for project_name, project_files in files_by_project.items():
-            logger.info(f"\nProcessing project: {project_name}")
-            
-            # Get imported files for this project
-            project_imported = self.imported_files.get(project_name, set())
-            new_files = [f for f in project_files if f not in project_imported]
-            
-            if not new_files:
-                logger.info(f"No new files for project {project_name}")
-                continue
-            
-            logger.info(f"Found {len(new_files)} new files for project {project_name}")
-            
-            # Setup collection for this project
-            collection_name = self.setup_collection(project_name)
-            
-            project_chunks = 0
-            for file_path in new_files:
-                logger.info(f"Processing: {file_path}")
-                
-                # Extract messages
-                messages = self.process_jsonl_file(file_path)
-                if not messages:
-                    logger.warning(f"No messages found in {file_path}")
-                    continue
-                
-                # Create conversation chunks
-                chunks = self.create_conversation_chunks(messages)
-                
-                # Import to project-specific collection
-                self.import_to_qdrant(chunks, collection_name)
-                
-                # Mark file as imported for this project
-                if project_name not in self.imported_files:
-                    self.imported_files[project_name] = set()
-                self.imported_files[project_name].add(file_path)
-                self.save_state()
-                
-                project_chunks += len(chunks)
-                logger.info(f"Imported {len(chunks)} chunks from {file_path}")
-            
-            total_chunks += project_chunks
-            logger.info(f"Project {project_name} complete: {project_chunks} chunks imported")
-        
-        logger.info(f"\nImport complete: {total_chunks} total chunks imported")
-        
-        # Show collection summary
-        logger.info("\nCollection summary:")
-        collections = self.client.get_collections().collections
-        for collection in collections:
-            if collection.name.startswith('conv_') or collection.name == 'conversations':
-                count = self.client.get_collection(collection.name).points_count
-                logger.info(f"  {collection.name}: {count} points")
-
-def main():
-    """Entry point for the importer."""
-    importer = ProjectAwareImporter()
-    importer.run()
-
-if __name__ == "__main__":
-    main()