claude-self-reflect 5.0.2 → 5.0.5

package/scripts/doctor.py

@@ -0,0 +1,342 @@
+#!/usr/bin/env python3
+"""
+Diagnostic script to check Claude Self-Reflect installation and identify issues.
+"""
+
+import os
+import sys
+import json
+import subprocess
+from pathlib import Path
+from typing import Dict, List, Tuple
+import urllib.request
+import urllib.error
+from datetime import datetime
+
+class Colors:
+    """Terminal colors for output"""
+    GREEN = '\033[92m'
+    YELLOW = '\033[93m'
+    RED = '\033[91m'
+    BLUE = '\033[94m'
+    ENDC = '\033[0m'
+    BOLD = '\033[1m'
+
+def print_header(text: str):
+    """Print a section header"""
+    print(f"\n{Colors.BOLD}{Colors.BLUE}{'='*60}{Colors.ENDC}")
+    print(f"{Colors.BOLD}{Colors.BLUE}{text}{Colors.ENDC}")
+    print(f"{Colors.BOLD}{Colors.BLUE}{'='*60}{Colors.ENDC}")
+
+def print_status(name: str, status: bool, message: str = ""):
+    """Print a status line with colored indicator"""
+    icon = f"{Colors.GREEN}✅{Colors.ENDC}" if status else f"{Colors.RED}❌{Colors.ENDC}"
+    status_text = f"{Colors.GREEN}OK{Colors.ENDC}" if status else f"{Colors.RED}FAILED{Colors.ENDC}"
+    print(f"{icon} {name}: {status_text}")
+    if message:
+        print(f"   {Colors.YELLOW}{message}{Colors.ENDC}")
+
+def check_docker() -> Tuple[bool, str]:
+    """Check if Docker is installed and running"""
+    try:
+        result = subprocess.run(['docker', 'info'], capture_output=True, text=True)
+        if result.returncode == 0:
+            # Check docker compose v2
+            compose_result = subprocess.run(['docker', 'compose', 'version'], capture_output=True, text=True)
+            if compose_result.returncode == 0:
+                return True, "Docker and Docker Compose v2 are running"
+            else:
+                return False, "Docker Compose v2 not found. Please update Docker Desktop"
+        else:
+            return False, "Docker is not running"
+    except FileNotFoundError:
+        return False, "Docker is not installed"
+
+def check_qdrant() -> Tuple[bool, str]:
+    """Check if Qdrant is running and accessible"""
+    try:
+        req = urllib.request.Request('http://localhost:6333')
+        with urllib.request.urlopen(req, timeout=5) as response:
+            if response.status == 200:
+                data = json.loads(response.read().decode())
+                version = data.get('version', 'unknown')
+                return True, f"Qdrant {version} is running on port 6333"
+    except:
+        pass
+    return False, "Qdrant is not accessible on localhost:6333"
+
+def check_collections() -> Tuple[bool, str, List[str]]:
+    """Check if Qdrant has any collections"""
+    try:
+        req = urllib.request.Request('http://localhost:6333/collections')
+        with urllib.request.urlopen(req, timeout=5) as response:
+            if response.status == 200:
+                data = json.loads(response.read().decode())
+                collections = data.get('result', {}).get('collections', [])
+                if collections:
+                    collection_names = [c['name'] for c in collections]
+                    return True, f"Found {len(collections)} collections", collection_names
+                else:
+                    return False, "No collections found - import may not have run", []
+    except:
+        pass
+    return False, "Could not query Qdrant collections", []
+
+def check_claude_projects() -> Tuple[bool, str, Dict]:
+    """Check Claude projects directory for JSONL files"""
+    claude_dir = Path.home() / '.claude' / 'projects'
+    stats = {
+        'total_projects': 0,
+        'total_files': 0,
+        'total_size': 0,
+        'sample_projects': []
+    }
+    
+    if not claude_dir.exists():
+        return False, f"Claude projects directory not found: {claude_dir}", stats
+    
+    try:
+        projects = list(claude_dir.iterdir())
+        for project in projects:
+            if project.is_dir():
+                jsonl_files = list(project.glob('*.jsonl'))
+                if jsonl_files:
+                    stats['total_projects'] += 1
+                    stats['total_files'] += len(jsonl_files)
+                    for f in jsonl_files:
+                        stats['total_size'] += f.stat().st_size
+                    if len(stats['sample_projects']) < 3:
+                        stats['sample_projects'].append(project.name)
+        
+        if stats['total_files'] == 0:
+            return False, "No JSONL files found in Claude projects", stats
+        
+        size_mb = stats['total_size'] / (1024 * 1024)
+        return True, f"Found {stats['total_files']} files across {stats['total_projects']} projects ({size_mb:.1f} MB)", stats
+    except Exception as e:
+        return False, f"Error scanning Claude projects: {e}", stats
+
+def check_import_state() -> Tuple[bool, str, Dict]:
+    """Check the import state file"""
+    config_dir = Path.home() / '.claude-self-reflect' / 'config'
+    state_file = config_dir / 'imported-files.json'
+    
+    stats = {
+        'imported_count': 0,
+        'last_import': None,
+        'has_metadata': False
+    }
+    
+    if not state_file.exists():
+        return False, "No import state file found - imports haven't run yet", stats
+    
+    try:
+        with open(state_file) as f:
+            state = json.load(f)
+        
+        imported = state.get('imported_files', {})
+        stats['imported_count'] = len(imported)
+        
+        # Check for metadata (new format)
+        for file_path, data in imported.items():
+            if isinstance(data, dict):
+                stats['has_metadata'] = True
+                if data.get('imported_at'):
+                    import_time = data['imported_at']
+                    if not stats['last_import'] or import_time > stats['last_import']:
+                        stats['last_import'] = import_time
+            elif isinstance(data, str):
+                # Old format
+                if not stats['last_import'] or data > stats['last_import']:
+                    stats['last_import'] = data
+        
+        if stats['imported_count'] == 0:
+            return False, "Import state exists but no files imported", stats
+        
+        msg = f"Imported {stats['imported_count']} files"
+        if stats['last_import']:
+            msg += f" (last: {stats['last_import'][:19]})"
+        if not stats['has_metadata']:
+            msg += " - OLD FORMAT (consider re-importing for metadata)"
+        
+        return True, msg, stats
+    except Exception as e:
+        return False, f"Error reading import state: {e}", stats
+
+def check_env_file() -> Tuple[bool, str, Dict]:
+    """Check .env file configuration"""
+    env_file = Path('.env')
+    config = {
+        'has_voyage_key': False,
+        'prefer_local': True,
+        'claude_logs_path': None,
+        'config_path': None
+    }
+    
+    if not env_file.exists():
+        return False, ".env file not found", config
+    
+    try:
+        with open(env_file) as f:
+            content = f.read()
+        
+        for line in content.split('\n'):
+            if '=' in line and not line.startswith('#'):
+                key, value = line.split('=', 1)
+                key = key.strip()
+                value = value.strip()
+                
+                if key == 'VOYAGE_KEY' and value and not value.startswith('your-'):
+                    config['has_voyage_key'] = True
+                elif key == 'PREFER_LOCAL_EMBEDDINGS':
+                    config['prefer_local'] = value.lower() == 'true'
+                elif key == 'CLAUDE_LOGS_PATH':
+                    config['claude_logs_path'] = value
+                elif key == 'CONFIG_PATH':
+                    config['config_path'] = value
+        
+        # Check critical paths
+        issues = []
+        if config['claude_logs_path'] and '~' in config['claude_logs_path']:
+            issues.append("CLAUDE_LOGS_PATH contains ~ which Docker won't expand")
+        if config['config_path'] and '~' in config['config_path']:
+            issues.append("CONFIG_PATH contains ~ which Docker won't expand")
+        
+        if issues:
+            return False, "; ".join(issues), config
+        
+        mode = "Local embeddings" if config['prefer_local'] else "Voyage AI embeddings"
+        return True, f"Configured for {mode}", config
+    except Exception as e:
+        return False, f"Error reading .env: {e}", config
+
+def check_docker_containers() -> Tuple[bool, str, List[str]]:
+    """Check which Docker containers are running"""
+    try:
+        result = subprocess.run(
+            ['docker', 'compose', 'ps', '--format', 'json'],
+            capture_output=True, text=True, cwd='.'
+        )
+        
+        if result.returncode != 0:
+            return False, "Could not query Docker containers", []
+        
+        running = []
+        lines = result.stdout.strip().split('\n')
+        for line in lines:
+            if line:
+                try:
+                    container = json.loads(line)
+                    if container.get('State') == 'running':
+                        running.append(container.get('Service', 'unknown'))
+                except:
+                    pass
+        
+        if not running:
+            return False, "No containers running", []
+        
+        essential = ['qdrant']
+        missing = [s for s in essential if s not in running]
+        
+        if missing:
+            return False, f"Essential services not running: {', '.join(missing)}", running
+        
+        return True, f"Running: {', '.join(running)}", running
+    except Exception as e:
+        return False, f"Error checking containers: {e}", []
+
+def main():
+    """Run all diagnostic checks"""
+    print(f"{Colors.BOLD}{Colors.BLUE}")
+    print("╔════════════════════════════════════════════════════════╗")
+    print("║     Claude Self-Reflect Diagnostic Tool v1.0          ║")
+    print("╚════════════════════════════════════════════════════════╝")
+    print(f"{Colors.ENDC}")
+    
+    # Basic checks
+    print_header("1. Environment Checks")
+    
+    docker_ok, docker_msg = check_docker()
+    print_status("Docker", docker_ok, docker_msg)
+    
+    env_ok, env_msg, env_config = check_env_file()
+    print_status("Environment (.env)", env_ok, env_msg)
+    
+    # Service checks
+    print_header("2. Service Status")
+    
+    containers_ok, containers_msg, running_containers = check_docker_containers()
+    print_status("Docker Containers", containers_ok, containers_msg)
+    
+    qdrant_ok, qdrant_msg = check_qdrant()
+    print_status("Qdrant Database", qdrant_ok, qdrant_msg)
+    
+    # Data checks
+    print_header("3. Data & Import Status")
+    
+    claude_ok, claude_msg, claude_stats = check_claude_projects()
+    print_status("Claude Projects", claude_ok, claude_msg)
+    if claude_stats['sample_projects']:
+        print(f"   Sample projects: {', '.join(claude_stats['sample_projects'][:3])}")
+    
+    import_ok, import_msg, import_stats = check_import_state()
+    print_status("Import State", import_ok, import_msg)
+    
+    collections_ok, collections_msg, collection_list = check_collections()
+    print_status("Qdrant Collections", collections_ok, collections_msg)
+    if collection_list:
+        print(f"   Collections: {', '.join(collection_list[:5])}")
+    
+    # Summary and recommendations
+    print_header("4. Summary & Recommendations")
+    
+    all_ok = all([docker_ok, env_ok, qdrant_ok, claude_ok])
+    
+    if all_ok and collections_ok:
+        print(f"{Colors.GREEN}✅ System appears to be working correctly!{Colors.ENDC}")
+    else:
+        print(f"{Colors.YELLOW}⚠️  Issues detected:{Colors.ENDC}")
+        
+        if not docker_ok:
+            print(f"\n{Colors.RED}Critical:{Colors.ENDC} Docker is required")
+            print("  → Install Docker Desktop from https://docker.com")
+        
+        if env_ok and '~' in str(env_config.get('claude_logs_path', '')):
+            print(f"\n{Colors.RED}Critical:{Colors.ENDC} Path expansion issue in .env")
+            print("  → Run: claude-self-reflect setup")
+            print("  → Or manually fix paths in .env to use full paths")
+        
+        if not qdrant_ok and docker_ok:
+            print(f"\n{Colors.YELLOW}Issue:{Colors.ENDC} Qdrant not running")
+            print("  → Run: docker compose --profile mcp up -d")
+        
+        if claude_ok and not collections_ok:
+            print(f"\n{Colors.YELLOW}Issue:{Colors.ENDC} No collections found but JSONL files exist")
+            print("  → Run: docker compose run --rm importer")
+            print("  → This will import your conversation history")
+        
+        if not claude_ok:
+            print(f"\n{Colors.YELLOW}Note:{Colors.ENDC} No Claude conversations found")
+            print("  → This is normal if you haven't used Claude Desktop yet")
+            print("  → The watcher will import new conversations automatically")
+    
+    # Quick commands
+    print_header("5. Quick Commands")
+    print("• Start services:  docker compose --profile mcp --profile watch up -d")
+    print("• Import conversations:  docker compose run --rm importer")
+    print("• View logs:  docker compose logs -f")
+    print("• Check status:  claude-self-reflect status")
+    print("• Restart everything:  docker compose down && docker compose --profile mcp --profile watch up -d")
+    
+    print(f"\n{Colors.BLUE}Documentation: https://github.com/ramakay/claude-self-reflect{Colors.ENDC}")
+    
+    # Return exit code based on critical issues
+    if not docker_ok:
+        sys.exit(1)
+    if all_ok:
+        sys.exit(0)
+    else:
+        sys.exit(2)  # Non-critical issues
+
+if __name__ == "__main__":
+    main()

package/scripts/embedding_service.py

@@ -0,0 +1,241 @@
+"""
+Embedding service abstraction to handle both local and cloud embeddings.
+Reduces complexity by separating embedding concerns from import logic.
+"""
+
+import os
+import logging
+from abc import ABC, abstractmethod
+from typing import List, Optional
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+class EmbeddingProvider(ABC):
+    """Abstract base class for embedding providers."""
+
+    @abstractmethod
+    def generate_embeddings(self, texts: List[str]) -> List[List[float]]:
+        """Generate embeddings for a list of texts."""
+        pass
+
+    @abstractmethod
+    def get_dimension(self) -> int:
+        """Get the dimension of embeddings produced by this provider."""
+        pass
+
+    @abstractmethod
+    def get_collection_suffix(self) -> str:
+        """Get the suffix for collection naming."""
+        pass
+
+
+class LocalEmbeddingProvider(EmbeddingProvider):
+    """Local embedding provider using FastEmbed."""
+
+    def __init__(self):
+        self.model = None
+        self.dimension = 384
+        self._initialize_model()
+
+    def _initialize_model(self):
+        """Initialize the FastEmbed model."""
+        try:
+            from fastembed import TextEmbedding
+            self.model = TextEmbedding(model_name="BAAI/bge-small-en-v1.5")
+            logger.info("Initialized local FastEmbed model (384 dimensions)")
+        except ImportError as e:
+            logger.error("FastEmbed not installed. Install with: pip install fastembed")
+            raise
+        except Exception as e:
+            logger.exception(f"Failed to initialize FastEmbed: {e}")
+            raise
+
+    def generate_embeddings(self, texts: List[str]) -> List[List[float]]:
+        """Generate embeddings using FastEmbed."""
+        if not self.model:
+            raise RuntimeError("FastEmbed model not initialized")
+
+        try:
+            embeddings = list(self.model.embed(texts))
+            return [list(emb) for emb in embeddings]
+        except Exception as e:
+            logger.error(f"Failed to generate local embeddings: {e}")
+            raise
+
+    def get_dimension(self) -> int:
+        """Get embedding dimension (384 for FastEmbed)."""
+        return self.dimension
+
+    def get_collection_suffix(self) -> str:
+        """Get collection suffix for local embeddings."""
+        return "local_384d"
+
+
+class CloudEmbeddingProvider(EmbeddingProvider):
+    """Cloud embedding provider using Voyage AI."""
+
+    def __init__(self, api_key: str):
+        # Don't store API key directly, use it only for client initialization
+        self.client = None
+        self.dimension = 1024
+        self._initialize_client(api_key)
+
+    def _initialize_client(self, api_key: str):
+        """Initialize the Voyage AI client."""
+        try:
+            import voyageai
+            self.client = voyageai.Client(api_key=api_key)
+            logger.info("Initialized Voyage AI client (1024 dimensions)")
+        except ImportError as e:
+            logger.error("voyageai not installed. Install with: pip install voyageai")
+            raise
+        except Exception as e:
+            logger.exception(f"Failed to initialize Voyage AI: {e}")
+            raise
+
+    def generate_embeddings(self, texts: List[str]) -> List[List[float]]:
+        """Generate embeddings using Voyage AI."""
+        if not self.client:
+            raise RuntimeError("Voyage AI client not initialized")
+
+        try:
+            result = self.client.embed(texts, model="voyage-2")
+            return result.embeddings
+        except Exception as e:
+            logger.error(f"Failed to generate cloud embeddings: {e}")
+            raise
+
+    def get_dimension(self) -> int:
+        """Get embedding dimension (1024 for Voyage)."""
+        return self.dimension
+
+    def get_collection_suffix(self) -> str:
+        """Get collection suffix for cloud embeddings."""
+        return "cloud_1024d"
+
+
+class EmbeddingService:
+    """
+    Service to manage embedding generation with automatic provider selection.
+    Reduces complexity by encapsulating embedding logic.
+    """
+
+    def __init__(self, prefer_local: bool = True, voyage_api_key: Optional[str] = None):
+        """
+        Initialize embedding service.
+
+        Args:
+            prefer_local: Whether to prefer local embeddings when available
+            voyage_api_key: API key for Voyage AI (if using cloud embeddings)
+        """
+        self.prefer_local = prefer_local
+        self.voyage_api_key = voyage_api_key
+        self.provider = None
+        self._initialize_provider()
+
+    def _initialize_provider(self):
+        """Initialize the appropriate embedding provider."""
+        if self.prefer_local or not self.voyage_api_key:
+            try:
+                self.provider = LocalEmbeddingProvider()
+                logger.info("Using local embedding provider (FastEmbed)")
+            except Exception as e:
+                logger.warning(f"Failed to initialize local provider: {e}")
+                if self.voyage_api_key:
+                    self._fallback_to_cloud()
+                else:
+                    raise RuntimeError("No embedding provider available")
+        else:
+            try:
+                self.provider = CloudEmbeddingProvider(self.voyage_api_key)
+                logger.info("Using cloud embedding provider (Voyage AI)")
+            except Exception as e:
+                logger.warning(f"Failed to initialize cloud provider: {e}")
+                self._fallback_to_local()
+
+    def _fallback_to_cloud(self):
+        """Fallback to cloud provider."""
+        if not self.voyage_api_key:
+            raise RuntimeError("No Voyage API key available for cloud fallback")
+        try:
+            self.provider = CloudEmbeddingProvider(self.voyage_api_key)
+            logger.info("Fallback to cloud embedding provider")
+            # Clear the key after use
+            self.voyage_api_key = None
+        except Exception as e:
+            raise RuntimeError(f"Failed to initialize any embedding provider: {e}")
+
+    def _fallback_to_local(self):
+        """Fallback to local provider."""
+        try:
+            self.provider = LocalEmbeddingProvider()
+            logger.info("Fallback to local embedding provider")
+        except Exception as e:
+            raise RuntimeError(f"Failed to initialize any embedding provider: {e}")
+
+    def generate_embeddings(self, texts: List[str]) -> List[List[float]]:
+        """
+        Generate embeddings for texts using the configured provider.
+
+        Args:
+            texts: List of texts to embed
+
+        Returns:
+            List of embedding vectors
+        """
+        if not self.provider:
+            raise RuntimeError("No embedding provider initialized")
+
+        # Filter out empty texts
+        non_empty_texts = [t for t in texts if t and t.strip()]
+        if not non_empty_texts:
+            return []
+
+        return self.provider.generate_embeddings(non_empty_texts)
+
+    def get_dimension(self) -> int:
+        """Get the dimension of embeddings."""
+        if not self.provider:
+            raise RuntimeError("No embedding provider initialized")
+        return self.provider.get_dimension()
+
+    def get_collection_suffix(self) -> str:
+        """Get the collection suffix for current provider."""
+        if not self.provider:
+            raise RuntimeError("No embedding provider initialized")
+        return self.provider.get_collection_suffix()
+
+    def get_provider_name(self) -> str:
+        """Get the name of the current provider."""
+        if isinstance(self.provider, LocalEmbeddingProvider):
+            return "FastEmbed (Local)"
+        elif isinstance(self.provider, CloudEmbeddingProvider):
+            return "Voyage AI (Cloud)"
+        else:
+            return "Unknown"
+
+
+# Factory function for convenience
+def create_embedding_service(
+    prefer_local: Optional[bool] = None,
+    voyage_api_key: Optional[str] = None
+) -> EmbeddingService:
+    """
+    Create an embedding service with environment variable defaults.
+
+    Args:
+        prefer_local: Override for PREFER_LOCAL_EMBEDDINGS env var
+        voyage_api_key: Override for VOYAGE_KEY env var
+
+    Returns:
+        Configured EmbeddingService instance
+    """
+    if prefer_local is None:
+        prefer_local = os.getenv("PREFER_LOCAL_EMBEDDINGS", "true").lower() == "true"
+
+    if voyage_api_key is None:
+        voyage_api_key = os.getenv("VOYAGE_KEY")
+
+    return EmbeddingService(prefer_local, voyage_api_key)