claude-self-reflect 3.0.0 → 3.0.1

package/scripts/importer/state/state_manager.py

@@ -0,0 +1,190 @@
+"""State management with atomic writes."""
+
+import json
+import logging
+import os
+from pathlib import Path
+from typing import Dict, Any, Optional, Set
+from datetime import datetime
+import fcntl
+import tempfile
+
+logger = logging.getLogger(__name__)
+
+
+class StateManager:
+    """
+    Manage import state with atomic writes and crash recovery.
+    
+    Implements the critical STATE_FILE fixes:
+    1. Handle empty dirname case
+    2. Atomic writes to prevent corruption
+    3. File locking for concurrent access
+    """
+    
+    def __init__(self, state_file: Path):
+        self.state_file = Path(state_file)
+        self.state: Dict[str, Any] = self._load_state()
+        self._lock_file = None
+    
+    def _load_state(self) -> Dict[str, Any]:
+        """Load state from file or create new."""
+        if self.state_file.exists():
+            try:
+                with open(self.state_file, 'r') as f:
+                    state = json.load(f)
+                    logger.debug(f"Loaded state with {len(state.get('processed', []))} processed files")
+                    return state
+            except Exception as e:
+                logger.error(f"Failed to load state: {e}")
+                # Create backup of corrupted state
+                backup = self.state_file.with_suffix('.corrupt')
+                try:
+                    self.state_file.rename(backup)
+                    logger.info(f"Backed up corrupted state to {backup}")
+                except Exception:
+                    pass
+        
+        # Return default state
+        return {
+            "processed": {},
+            "failed": {},
+            "last_updated": None,
+            "version": "3.0.0"
+        }
+    
+    def save_state(self) -> None:
+        """
+        Save state with atomic write.
+        
+        Implements the critical fix for STATE_FILE directory handling.
+        """
+        try:
+            # FIX: Handle case where STATE_FILE has no directory component
+            state_dir = self.state_file.parent
+            if state_dir and state_dir != Path('.'):
+                state_dir.mkdir(parents=True, exist_ok=True)
+            
+            # Update timestamp
+            self.state["last_updated"] = datetime.now().isoformat()
+            
+            # Use atomic write to prevent corruption during crashes
+            # Create temp file in same directory for atomic rename
+            temp_fd, temp_path = tempfile.mkstemp(
+                dir=str(state_dir) if state_dir else '.',
+                prefix='.tmp_state_',
+                suffix='.json'
+            )
+            
+            try:
+                # Write to temp file
+                with os.fdopen(temp_fd, 'w') as f:
+                    json.dump(self.state, f, indent=2)
+                
+                # Atomic rename (on POSIX systems)
+                Path(temp_path).replace(self.state_file)
+                
+                logger.debug("State saved successfully")
+                
+            except Exception as e:
+                # Clean up temp file on error
+                try:
+                    os.unlink(temp_path)
+                except:
+                    pass
+                raise e
+                
+        except Exception as e:
+            logger.error(f"Failed to save state: {e}")
+            raise
+    
+    def is_processed(self, file_path: Path) -> bool:
+        """Check if file has been processed."""
+        return str(file_path) in self.state.get("processed", {})
+    
+    def mark_processed(self, file_path: Path, points_created: int) -> None:
+        """Mark file as processed."""
+        if "processed" not in self.state:
+            self.state["processed"] = {}
+        
+        self.state["processed"][str(file_path)] = {
+            "timestamp": datetime.now().isoformat(),
+            "points_created": points_created
+        }
+        
+        # Remove from failed if present
+        if str(file_path) in self.state.get("failed", {}):
+            del self.state["failed"][str(file_path)]
+        
+        self.save_state()
+    
+    def mark_failed(self, file_path: Path, error: str) -> None:
+        """Mark file as failed."""
+        if "failed" not in self.state:
+            self.state["failed"] = {}
+        
+        self.state["failed"][str(file_path)] = {
+            "timestamp": datetime.now().isoformat(),
+            "error": error
+        }
+        
+        self.save_state()
+    
+    def get_processed_files(self) -> Set[str]:
+        """Get set of processed file paths."""
+        return set(self.state.get("processed", {}).keys())
+    
+    def get_failed_files(self) -> Set[str]:
+        """Get set of failed file paths."""
+        return set(self.state.get("failed", {}).keys())
+    
+    def reset(self) -> None:
+        """Reset state to empty."""
+        self.state = {
+            "processed": {},
+            "failed": {},
+            "last_updated": datetime.now().isoformat(),
+            "version": "3.0.0"
+        }
+        self.save_state()
+    
+    def get_statistics(self) -> Dict[str, Any]:
+        """Get import statistics."""
+        processed = self.state.get("processed", {})
+        failed = self.state.get("failed", {})
+        
+        total_points = sum(
+            info.get("points_created", 0) 
+            for info in processed.values()
+        )
+        
+        return {
+            "processed_count": len(processed),
+            "failed_count": len(failed),
+            "total_points": total_points,
+            "last_updated": self.state.get("last_updated"),
+            "version": self.state.get("version")
+        }
+    
+    def acquire_lock(self) -> bool:
+        """Acquire exclusive lock for state file (Unix/Linux only)."""
+        if os.name == 'nt':  # Windows
+            logger.warning("File locking not implemented for Windows")
+            return True  # Allow operation to proceed without locking on Windows
+        try:
+            lock_path = self.state_file.with_suffix('.lock')
+            self._lock_file = open(lock_path, 'w')
+            fcntl.flock(self._lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+            return True
+        except (IOError, OSError):
+            return False
+    
+    def release_lock(self) -> None:
+        """Release state file lock."""
+        if self._lock_file:
+            try:
+                fcntl.flock(self._lock_file.fileno(), fcntl.LOCK_UN)
+                self._lock_file.close()
+            except:
+                pass
+            self._lock_file = None

package/scripts/importer/storage/__init__.py

@@ -0,0 +1,5 @@
+"""Storage backend for vector database operations."""
+
+from .qdrant_storage import QdrantStorage
+
+__all__ = ["QdrantStorage"]

package/scripts/importer/storage/qdrant_storage.py

@@ -0,0 +1,250 @@
+"""Qdrant vector database storage implementation."""
+
+import logging
+from typing import List, Optional, Dict, Any
+from qdrant_client import QdrantClient
+from qdrant_client.models import (
+    Distance,
+    VectorParams,
+    PointStruct,
+    CollectionInfo
+)
+
+from ..core import ProcessedPoint
+from ..core.exceptions import StorageError
+
+logger = logging.getLogger(__name__)
+
+
+class QdrantStorage:
+    """
+    Qdrant storage backend implementation.
+    
+    Handles all interactions with the Qdrant vector database.
+    """
+    
+    def __init__(self, url: str = "http://localhost:6333", api_key: Optional[str] = None):
+        self.url = url
+        self.api_key = api_key
+        self.client = None
+        self._initialized = False
+    
+    def initialize(self) -> None:
+        """Initialize connection to Qdrant."""
+        try:
+            self.client = QdrantClient(
+                url=self.url,
+                api_key=self.api_key,
+                timeout=30
+            )
+            
+            # Test connection
+            self.client.get_collections()
+            self._initialized = True
+            logger.info(f"Connected to Qdrant at {self.url}")
+            
+        except Exception as e:
+            raise StorageError(
+                operation="initialize",
+                collection="N/A",
+                reason=f"Failed to connect to Qdrant: {e}"
+            )
+    
+    def create_collection(self, name: str, dimension: int) -> bool:
+        """
+        Create a new collection if it doesn't exist.
+        
+        Args:
+            name: Collection name
+            dimension: Vector dimension
+            
+        Returns:
+            True if created, False if already exists
+        """
+        if not self._initialized:
+            raise StorageError(
+                operation="create_collection",
+                collection=name,
+                reason="Storage not initialized"
+            )
+        
+        try:
+            if self.collection_exists(name):
+                logger.debug(f"Collection {name} already exists")
+                return False
+            
+            self.client.create_collection(
+                collection_name=name,
+                vectors_config=VectorParams(
+                    size=dimension,
+                    distance=Distance.COSINE
+                )
+            )
+            
+            logger.info(f"Created collection {name} with dimension {dimension}")
+            return True
+            
+        except Exception as e:
+            raise StorageError(
+                operation="create_collection",
+                collection=name,
+                reason=str(e)
+            )
+    
+    def collection_exists(self, name: str) -> bool:
+        """Check if a collection exists."""
+        if not self._initialized:
+            raise StorageError(
+                operation="collection_exists",
+                collection=name,
+                reason="Storage not initialized"
+            )
+        
+        try:
+            collections = self.client.get_collections().collections
+            return any(c.name == name for c in collections)
+        except Exception as e:
+            logger.error(f"Failed to check collection existence: {e}")
+            return False
+    
+    def upsert_points(self, collection: str, points: List[ProcessedPoint]) -> int:
+        """
+        Insert or update points in a collection.
+        
+        Args:
+            collection: Collection name
+            points: List of points to upsert
+            
+        Returns:
+            Number of points upserted
+        """
+        if not self._initialized:
+            raise StorageError(
+                operation="upsert_points",
+                collection=collection,
+                reason="Storage not initialized"
+            )
+        
+        if not points:
+            return 0
+        
+        try:
+            # Ensure collection exists with correct dimension
+            if not self.collection_exists(collection):
+                dimension = len(points[0].vector)
+                self.create_collection(collection, dimension)
+            
+            # Convert to Qdrant points
+            qdrant_points = [
+                PointStruct(
+                    id=self._generate_point_id(point.id),
+                    vector=point.vector,
+                    payload=point.payload
+                )
+                for point in points
+            ]
+            
+            # Batch upsert
+            operation_info = self.client.upsert(
+                collection_name=collection,
+                points=qdrant_points
+            )
+            
+            logger.debug(f"Upserted {len(points)} points to {collection}")
+            return len(points)
+            
+        except Exception as e:
+            raise StorageError(
+                operation="upsert_points",
+                collection=collection,
+                reason=str(e)
+            )
+    
+    def get_collection_info(self, name: str) -> Optional[CollectionInfo]:
+        """Get information about a collection."""
+        if not self._initialized:
+            return None
+        
+        try:
+            return self.client.get_collection(collection_name=name)
+        except Exception as e:
+            logger.error(f"Failed to get collection info: {e}")
+            return None
+    
+    def delete_collection(self, name: str) -> bool:
+        """Delete a collection."""
+        if not self._initialized:
+            raise StorageError(
+                operation="delete_collection",
+                collection=name,
+                reason="Storage not initialized"
+            )
+        
+        try:
+            if not self.collection_exists(name):
+                return False
+            
+            self.client.delete_collection(collection_name=name)
+            logger.info(f"Deleted collection {name}")
+            return True
+            
+        except Exception as e:
+            raise StorageError(
+                operation="delete_collection",
+                collection=name,
+                reason=str(e)
+            )
+    
+    def get_collections(self) -> List[str]:
+        """Get list of all collection names."""
+        if not self._initialized:
+            return []
+        
+        try:
+            collections = self.client.get_collections().collections
+            return [c.name for c in collections]
+        except Exception as e:
+            logger.error(f"Failed to get collections: {e}")
+            return []
+    
+    def count_points(self, collection: str) -> int:
+        """Count points in a collection."""
+        if not self._initialized:
+            return 0
+        
+        try:
+            info = self.get_collection_info(collection)
+            return info.points_count if info else 0
+        except Exception as e:
+            logger.error(f"Failed to count points: {e}")
+            return 0
+    
+    def _generate_point_id(self, string_id: str) -> str:
+        """
+        Generate a valid point ID from a string.
+        
+        Qdrant accepts string IDs directly in newer versions.
+        For older versions, we might need to hash to integer.
+        """
+        # For now, use string IDs directly
+        # If needed, can hash: int(hashlib.md5(string_id.encode()).hexdigest()[:16], 16)
+        return string_id
+    
+    def health_check(self) -> Dict[str, Any]:
+        """Check health of Qdrant connection."""
+        if not self._initialized:
+            return {"healthy": False, "reason": "Not initialized"}
+        
+        try:
+            collections = self.client.get_collections()
+            return {
+                "healthy": True,
+                "url": self.url,
+                "collections_count": len(collections.collections),
+                "total_points": sum(
+                    self.count_points(c.name) 
+                    for c in collections.collections
+                )
+            }
+        except Exception as e:
+            return {"healthy": False, "reason": str(e)}

package/scripts/importer/utils/__init__.py

@@ -0,0 +1,9 @@
+"""Utility functions for the import system."""
+
+from .project_normalizer import ProjectNormalizer
+from .logger import setup_logging
+
+__all__ = [
+    "ProjectNormalizer",
+    "setup_logging"
+]

package/scripts/importer/utils/logger.py

@@ -0,0 +1,87 @@
+"""Centralized logging configuration."""
+
+import logging
+import sys
+from pathlib import Path
+from datetime import datetime
+from typing import Optional
+
+
+def setup_logging(
+    level: str = "INFO",
+    log_file: Optional[str] = None,
+    format_string: Optional[str] = None
+) -> logging.Logger:
+    """
+    Set up logging configuration.
+    
+    Args:
+        level: Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
+        log_file: Optional log file path
+        format_string: Optional custom format string
+        
+    Returns:
+        Configured root logger
+    """
+    # Default format
+    if not format_string:
+        format_string = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+    
+    # Configure root logger
+    logging.basicConfig(
+        level=getattr(logging, level.upper()),
+        format=format_string,
+        handlers=[]  # Clear default handlers
+    )
+    
+    logger = logging.getLogger()
+    
+    # Console handler
+    console_handler = logging.StreamHandler(sys.stdout)
+    console_handler.setFormatter(logging.Formatter(format_string))
+    logger.addHandler(console_handler)
+    
+    # File handler if specified
+    if log_file:
+        log_path = Path(log_file)
+        log_path.parent.mkdir(parents=True, exist_ok=True)
+        
+        file_handler = logging.FileHandler(log_path)
+        file_handler.setFormatter(logging.Formatter(format_string))
+        logger.addHandler(file_handler)
+    
+    return logger
+
+
+class ProgressLogger:
+    """Logger for tracking import progress."""
+    
+    def __init__(self, total: int, logger: Optional[logging.Logger] = None):
+        self.total = total
+        self.current = 0
+        self.logger = logger or logging.getLogger(__name__)
+        self.start_time = datetime.now()
+    
+    def update(self, increment: int = 1, message: Optional[str] = None) -> None:
+        """Update progress."""
+        self.current += increment
+        percentage = (self.current / self.total * 100) if self.total > 0 else 0
+        
+        elapsed = (datetime.now() - self.start_time).total_seconds()
+        rate = self.current / elapsed if elapsed > 0 else 0
+        eta = (self.total - self.current) / rate if rate > 0 else 0
+        
+        log_message = f"Progress: {self.current}/{self.total} ({percentage:.1f}%)"
+        if message:
+            log_message += f" - {message}"
+        log_message += f" - Rate: {rate:.1f}/s - ETA: {eta:.0f}s"
+        
+        self.logger.info(log_message)
+    
+    def complete(self) -> None:
+        """Mark as complete."""
+        elapsed = (datetime.now() - self.start_time).total_seconds()
+        self.logger.info(
+            f"Completed {self.total} items in {elapsed:.1f}s "
+            f"({self.total/elapsed:.1f} items/s)"
+        )

package/scripts/importer/utils/project_normalizer.py

@@ -0,0 +1,120 @@
+"""Project name normalization utilities."""
+
+import hashlib
+import logging
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+class ProjectNormalizer:
+    """
+    Normalize project names and generate collection names.
+    
+    This is the CRITICAL component that was broken before.
+    Must correctly extract project names from Claude's dash-separated format.
+    """
+    
+    @staticmethod
+    def normalize_project_name(project_path: str) -> str:
+        """
+        Normalize a project path to a consistent project name.
+        
+        CRITICAL: This must match the implementation in utils.py
+        
+        Examples:
+        - "-Users-name-projects-claude-self-reflect" -> "claude-self-reflect"
+        - "claude-self-reflect" -> "claude-self-reflect"
+        - "/path/to/-Users-name-projects-myapp" -> "myapp"
+        """
+        # Get the final component of the path
+        if '/' in project_path:
+            final_component = project_path.split('/')[-1]
+        else:
+            final_component = project_path
+        
+        # Handle Claude's dash-separated format
+        if final_component.startswith('-') and 'projects' in final_component:
+            # Find the last occurrence of 'projects-'
+            idx = final_component.rfind('projects-')
+            if idx != -1:
+                # Extract everything after 'projects-'
+                project_name = final_component[idx + len('projects-'):]
+                logger.debug(f"Normalized '{project_path}' to '{project_name}'")
+                return project_name
+        
+        # Already normalized or different format
+        logger.debug(f"Project path '{project_path}' already normalized")
+        return final_component
+    
+    def get_project_name(self, file_path: Path) -> str:
+        """
+        Extract project name from a file path.
+        
+        Args:
+            file_path: Path to a conversation file
+            
+        Returns:
+            Normalized project name
+        """
+        # Get the parent directory name
+        parent_name = file_path.parent.name
+        
+        # Normalize it
+        return self.normalize_project_name(parent_name)
+    
+    def get_collection_name(self, file_path: Path) -> str:
+        """
+        Generate collection name for a file.
+        
+        Format: conv_HASH_local
+        Where HASH is first 8 chars of MD5 hash of normalized project name.
+        """
+        project_name = self.get_project_name(file_path)
+        
+        # Generate hash
+        project_hash = hashlib.md5(project_name.encode()).hexdigest()[:8]
+        
+        # Generate collection name
+        collection_name = f"conv_{project_hash}_local"
+        
+        logger.debug(f"Collection for project '{project_name}': {collection_name}")
+        return collection_name
+    
+    @staticmethod
+    def validate_normalization() -> bool:
+        """
+        Self-test to ensure normalization is working correctly.
+        
+        Returns:
+            True if all tests pass
+        """
+        test_cases = [
+            ("-Users-name-projects-claude-self-reflect", "claude-self-reflect", "7f6df0fc"),
+            ("claude-self-reflect", "claude-self-reflect", "7f6df0fc"),
+            ("/Users/name/.claude/projects/-Users-name-projects-myapp", "myapp", None),
+            ("-Users-name-projects-procsolve-website", "procsolve-website", "9f2f312b")
+        ]
+        
+        normalizer = ProjectNormalizer()
+        all_passed = True
+        
+        for input_path, expected_name, expected_hash in test_cases:
+            normalized = normalizer.normalize_project_name(input_path)
+            if normalized != expected_name:
+                logger.error(
+                    f"Normalization failed: '{input_path}' -> '{normalized}' "
+                    f"(expected '{expected_name}')"
+                )
+                all_passed = False
+            
+            if expected_hash:
+                actual_hash = hashlib.md5(normalized.encode()).hexdigest()[:8]
+                if actual_hash != expected_hash:
+                    logger.error(
+                        f"Hash mismatch for '{normalized}': "
+                        f"{actual_hash} != {expected_hash}"
+                    )
+                    all_passed = False
+        
+        return all_passed