claude-self-reflect 3.0.0 → 3.0.2

package/scripts/importer/processors/tool_extractor.py

@@ -0,0 +1,165 @@
+"""Extract tool usage and file references from conversations."""
+
+import re
+import logging
+from typing import Dict, Any, Set, List
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+class ToolUsageExtractor:
+    """Extract files analyzed, edited, and tools used from conversations."""
+    
+    def __init__(self):
+        # Patterns for file operations
+        self.file_patterns = {
+            'analyzed': [
+                re.compile(r'(?:reading|analyzing|examining|looking at|checking)\s+(?:file\s+)?([/\w\-\.]+\.\w+)', re.IGNORECASE),
+                re.compile(r'(?:Read|read)\s+([/\w\-\.]+\.\w+)', re.IGNORECASE),
+                re.compile(r'(?:in|from)\s+file\s+([/\w\-\.]+\.\w+)', re.IGNORECASE)
+            ],
+            'edited': [
+                re.compile(r'(?:editing|modifying|updating|changing|writing to)\s+(?:file\s+)?([/\w\-\.]+\.\w+)', re.IGNORECASE),
+                re.compile(r'(?:Edit|Write)\s+([/\w\-\.]+\.\w+)', re.IGNORECASE),
+                re.compile(r'(?:changes to|modified|updated)\s+([/\w\-\.]+\.\w+)', re.IGNORECASE)
+            ],
+            'created': [
+                re.compile(r'(?:creating|created|new file)\s+([/\w\-\.]+\.\w+)', re.IGNORECASE),
+                re.compile(r'(?:Write|Create)\s+new\s+file\s+([/\w\-\.]+\.\w+)', re.IGNORECASE)
+            ]
+        }
+        
+        # Tool patterns
+        self.tool_patterns = [
+            re.compile(r'(?:using|running|executing)\s+(\w+)\s+(?:tool|command)', re.IGNORECASE),
+            re.compile(r'(?:Tool:|Command:)\s*(\w+)', re.IGNORECASE),
+            re.compile(r'```(?:bash|shell|sh)\n([a-z]+)', re.IGNORECASE),
+            re.compile(r'\$\s+([a-z]+)\s+', re.IGNORECASE),  # Command line
+            re.compile(r'(?:npm|yarn|pip|cargo|go|cargo)\s+([\w\-]+)', re.IGNORECASE)
+        ]
+        
+        # MCP tool pattern
+        self.mcp_pattern = re.compile(r'mcp__([a-zA-Z0-9\-_]+)__([a-zA-Z0-9\-_]+)')
+    
+    def extract(self, text: str) -> Dict[str, Any]:
+        """
+        Extract tool usage information from text.
+        
+        Returns:
+            Dictionary with files_analyzed, files_edited, tools_used
+        """
+        files_analyzed = set()
+        files_edited = set()
+        files_created = set()
+        tools_used = set()
+        
+        # Extract file operations
+        for pattern in self.file_patterns['analyzed']:
+            matches = pattern.findall(text)
+            for match in matches:
+                file_path = self._normalize_file_path(match)
+                if file_path:
+                    files_analyzed.add(file_path)
+        
+        for pattern in self.file_patterns['edited']:
+            matches = pattern.findall(text)
+            for match in matches:
+                file_path = self._normalize_file_path(match)
+                if file_path:
+                    files_edited.add(file_path)
+        
+        for pattern in self.file_patterns['created']:
+            matches = pattern.findall(text)
+            for match in matches:
+                file_path = self._normalize_file_path(match)
+                if file_path:
+                    files_created.add(file_path)
+        
+        # Extract tools
+        for pattern in self.tool_patterns:
+            matches = pattern.findall(text)
+            for match in matches:
+                tool = match.lower().strip()
+                if self._is_valid_tool(tool):
+                    tools_used.add(tool)
+        
+        # Extract MCP tools specifically
+        mcp_matches = self.mcp_pattern.findall(text)
+        for server, tool in mcp_matches:
+            tools_used.add(f"mcp:{server}:{tool}")
+        
+        # Look for common CLI tools
+        common_tools = [
+            'git', 'npm', 'yarn', 'pip', 'python', 'node', 'docker',
+            'kubectl', 'aws', 'gcloud', 'az', 'terraform', 'ansible',
+            'make', 'gradle', 'maven', 'cargo', 'go', 'rustc'
+        ]
+        for tool in common_tools:
+            if re.search(rf'\b{tool}\b', text, re.IGNORECASE):
+                tools_used.add(tool)
+        
+        # Combine all files for backward compatibility
+        all_files = files_analyzed | files_edited | files_created
+        
+        return {
+            "files_analyzed": list(files_analyzed)[:50],
+            "files_edited": list(files_edited)[:50],
+            "files_created": list(files_created)[:50],
+            "files": list(all_files)[:50],  # Legacy field
+            "tools_used": list(tools_used)[:30],
+            "file_count": len(all_files),
+            "tool_count": len(tools_used)
+        }
+    
+    def _normalize_file_path(self, path: str) -> str:
+        """Normalize and validate file path."""
+        # Remove quotes and whitespace
+        path = path.strip('\'"` \n')
+        
+        # Skip if too short or too long
+        if len(path) < 3 or len(path) > 200:
+            return ""
+        
+        # Must have an extension
+        if '.' not in path:
+            return ""
+        
+        # Extract just the filename if it's a full path
+        if '/' in path:
+            # Get the last component
+            path = path.split('/')[-1]
+        
+        # Validate extension
+        valid_extensions = {
+            'py', 'js', 'ts', 'jsx', 'tsx', 'java', 'go', 'rs', 'cpp', 'c', 'h',
+            'md', 'txt', 'json', 'yaml', 'yml', 'xml', 'html', 'css', 'scss',
+            'sql', 'sh', 'bash', 'dockerfile', 'makefile', 'toml', 'ini', 'cfg'
+        }
+        
+        ext = path.split('.')[-1].lower()
+        if ext not in valid_extensions:
+            return ""
+        
+        return path
+    
+    def _is_valid_tool(self, tool: str) -> bool:
+        """Check if a string is a valid tool name."""
+        # Skip common words
+        skip_words = {
+            'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
+            'this', 'that', 'with', 'from', 'as', 'is', 'was', 'are', 'were'
+        }
+        
+        if tool in skip_words:
+            return False
+        
+        # Must be alphanumeric with possible hyphens/underscores
+        if not re.match(r'^[a-z0-9\-_]+$', tool):
+            return False
+        
+        # Reasonable length
+        if len(tool) < 2 or len(tool) > 30:
+            return False
+        
+        return True

package/scripts/importer/state/__init__.py

@@ -0,0 +1,5 @@
+"""State management for import tracking."""
+
+from .state_manager import StateManager
+
+__all__ = ["StateManager"]

package/scripts/importer/state/state_manager.py

@@ -0,0 +1,190 @@
+"""State management with atomic writes."""
+
+import json
+import logging
+import os
+from pathlib import Path
+from typing import Dict, Any, Optional, Set
+from datetime import datetime
+import fcntl
+import tempfile
+
+logger = logging.getLogger(__name__)
+
+
+class StateManager:
+    """
+    Manage import state with atomic writes and crash recovery.
+    
+    Implements the critical STATE_FILE fixes:
+    1. Handle empty dirname case
+    2. Atomic writes to prevent corruption
+    3. File locking for concurrent access
+    """
+    
+    def __init__(self, state_file: Path):
+        self.state_file = Path(state_file)
+        self.state: Dict[str, Any] = self._load_state()
+        self._lock_file = None
+    
+    def _load_state(self) -> Dict[str, Any]:
+        """Load state from file or create new."""
+        if self.state_file.exists():
+            try:
+                with open(self.state_file, 'r') as f:
+                    state = json.load(f)
+                    logger.debug(f"Loaded state with {len(state.get('processed', []))} processed files")
+                    return state
+            except Exception as e:
+                logger.error(f"Failed to load state: {e}")
+                # Create backup of corrupted state
+                backup = self.state_file.with_suffix('.corrupt')
+                try:
+                    self.state_file.rename(backup)
+                    logger.info(f"Backed up corrupted state to {backup}")
+                except Exception:
+                    pass
+        
+        # Return default state
+        return {
+            "processed": {},
+            "failed": {},
+            "last_updated": None,
+            "version": "3.0.0"
+        }
+    
+    def save_state(self) -> None:
+        """
+        Save state with atomic write.
+        
+        Implements the critical fix for STATE_FILE directory handling.
+        """
+        try:
+            # FIX: Handle case where STATE_FILE has no directory component
+            state_dir = self.state_file.parent
+            if state_dir and state_dir != Path('.'):
+                state_dir.mkdir(parents=True, exist_ok=True)
+            
+            # Update timestamp
+            self.state["last_updated"] = datetime.now().isoformat()
+            
+            # Use atomic write to prevent corruption during crashes
+            # Create temp file in same directory for atomic rename
+            temp_fd, temp_path = tempfile.mkstemp(
+                dir=str(state_dir) if state_dir else '.',
+                prefix='.tmp_state_',
+                suffix='.json'
+            )
+            
+            try:
+                # Write to temp file
+                with os.fdopen(temp_fd, 'w') as f:
+                    json.dump(self.state, f, indent=2)
+                
+                # Atomic rename (on POSIX systems)
+                Path(temp_path).replace(self.state_file)
+                
+                logger.debug("State saved successfully")
+                
+            except Exception as e:
+                # Clean up temp file on error
+                try:
+                    os.unlink(temp_path)
+                except:
+                    pass
+                raise e
+                
+        except Exception as e:
+            logger.error(f"Failed to save state: {e}")
+            raise
+    
+    def is_processed(self, file_path: Path) -> bool:
+        """Check if file has been processed."""
+        return str(file_path) in self.state.get("processed", {})
+    
+    def mark_processed(self, file_path: Path, points_created: int) -> None:
+        """Mark file as processed."""
+        if "processed" not in self.state:
+            self.state["processed"] = {}
+        
+        self.state["processed"][str(file_path)] = {
+            "timestamp": datetime.now().isoformat(),
+            "points_created": points_created
+        }
+        
+        # Remove from failed if present
+        if str(file_path) in self.state.get("failed", {}):
+            del self.state["failed"][str(file_path)]
+        
+        self.save_state()
+    
+    def mark_failed(self, file_path: Path, error: str) -> None:
+        """Mark file as failed."""
+        if "failed" not in self.state:
+            self.state["failed"] = {}
+        
+        self.state["failed"][str(file_path)] = {
+            "timestamp": datetime.now().isoformat(),
+            "error": error
+        }
+        
+        self.save_state()
+    
+    def get_processed_files(self) -> Set[str]:
+        """Get set of processed file paths."""
+        return set(self.state.get("processed", {}).keys())
+    
+    def get_failed_files(self) -> Set[str]:
+        """Get set of failed file paths."""
+        return set(self.state.get("failed", {}).keys())
+    
+    def reset(self) -> None:
+        """Reset state to empty."""
+        self.state = {
+            "processed": {},
+            "failed": {},
+            "last_updated": datetime.now().isoformat(),
+            "version": "3.0.0"
+        }
+        self.save_state()
+    
+    def get_statistics(self) -> Dict[str, Any]:
+        """Get import statistics."""
+        processed = self.state.get("processed", {})
+        failed = self.state.get("failed", {})
+        
+        total_points = sum(
+            info.get("points_created", 0) 
+            for info in processed.values()
+        )
+        
+        return {
+            "processed_count": len(processed),
+            "failed_count": len(failed),
+            "total_points": total_points,
+            "last_updated": self.state.get("last_updated"),
+            "version": self.state.get("version")
+        }
+    
+    def acquire_lock(self) -> bool:
+        """Acquire exclusive lock for state file (Unix/Linux only)."""
+        if os.name == 'nt':  # Windows
+            logger.warning("File locking not implemented for Windows")
+            return True  # Allow operation to proceed without locking on Windows
+        try:
+            lock_path = self.state_file.with_suffix('.lock')
+            self._lock_file = open(lock_path, 'w')
+            fcntl.flock(self._lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+            return True
+        except (IOError, OSError):
+            return False
+    
+    def release_lock(self) -> None:
+        """Release state file lock."""
+        if self._lock_file:
+            try:
+                fcntl.flock(self._lock_file.fileno(), fcntl.LOCK_UN)
+                self._lock_file.close()
+            except:
+                pass
+            self._lock_file = None

package/scripts/importer/storage/__init__.py

@@ -0,0 +1,5 @@
+"""Storage backend for vector database operations."""
+
+from .qdrant_storage import QdrantStorage
+
+__all__ = ["QdrantStorage"]

package/scripts/importer/storage/qdrant_storage.py

@@ -0,0 +1,250 @@
+"""Qdrant vector database storage implementation."""
+
+import logging
+from typing import List, Optional, Dict, Any
+from qdrant_client import QdrantClient
+from qdrant_client.models import (
+    Distance,
+    VectorParams,
+    PointStruct,
+    CollectionInfo
+)
+
+from ..core import ProcessedPoint
+from ..core.exceptions import StorageError
+
+logger = logging.getLogger(__name__)
+
+
+class QdrantStorage:
+    """
+    Qdrant storage backend implementation.
+    
+    Handles all interactions with the Qdrant vector database.
+    """
+    
+    def __init__(self, url: str = "http://localhost:6333", api_key: Optional[str] = None):
+        self.url = url
+        self.api_key = api_key
+        self.client = None
+        self._initialized = False
+    
+    def initialize(self) -> None:
+        """Initialize connection to Qdrant."""
+        try:
+            self.client = QdrantClient(
+                url=self.url,
+                api_key=self.api_key,
+                timeout=30
+            )
+            
+            # Test connection
+            self.client.get_collections()
+            self._initialized = True
+            logger.info(f"Connected to Qdrant at {self.url}")
+            
+        except Exception as e:
+            raise StorageError(
+                operation="initialize",
+                collection="N/A",
+                reason=f"Failed to connect to Qdrant: {e}"
+            )
+    
+    def create_collection(self, name: str, dimension: int) -> bool:
+        """
+        Create a new collection if it doesn't exist.
+        
+        Args:
+            name: Collection name
+            dimension: Vector dimension
+            
+        Returns:
+            True if created, False if already exists
+        """
+        if not self._initialized:
+            raise StorageError(
+                operation="create_collection",
+                collection=name,
+                reason="Storage not initialized"
+            )
+        
+        try:
+            if self.collection_exists(name):
+                logger.debug(f"Collection {name} already exists")
+                return False
+            
+            self.client.create_collection(
+                collection_name=name,
+                vectors_config=VectorParams(
+                    size=dimension,
+                    distance=Distance.COSINE
+                )
+            )
+            
+            logger.info(f"Created collection {name} with dimension {dimension}")
+            return True
+            
+        except Exception as e:
+            raise StorageError(
+                operation="create_collection",
+                collection=name,
+                reason=str(e)
+            )
+    
+    def collection_exists(self, name: str) -> bool:
+        """Check if a collection exists."""
+        if not self._initialized:
+            raise StorageError(
+                operation="collection_exists",
+                collection=name,
+                reason="Storage not initialized"
+            )
+        
+        try:
+            collections = self.client.get_collections().collections
+            return any(c.name == name for c in collections)
+        except Exception as e:
+            logger.error(f"Failed to check collection existence: {e}")
+            return False
+    
+    def upsert_points(self, collection: str, points: List[ProcessedPoint]) -> int:
+        """
+        Insert or update points in a collection.
+        
+        Args:
+            collection: Collection name
+            points: List of points to upsert
+            
+        Returns:
+            Number of points upserted
+        """
+        if not self._initialized:
+            raise StorageError(
+                operation="upsert_points",
+                collection=collection,
+                reason="Storage not initialized"
+            )
+        
+        if not points:
+            return 0
+        
+        try:
+            # Ensure collection exists with correct dimension
+            if not self.collection_exists(collection):
+                dimension = len(points[0].vector)
+                self.create_collection(collection, dimension)
+            
+            # Convert to Qdrant points
+            qdrant_points = [
+                PointStruct(
+                    id=self._generate_point_id(point.id),
+                    vector=point.vector,
+                    payload=point.payload
+                )
+                for point in points
+            ]
+            
+            # Batch upsert
+            operation_info = self.client.upsert(
+                collection_name=collection,
+                points=qdrant_points
+            )
+            
+            logger.debug(f"Upserted {len(points)} points to {collection}")
+            return len(points)
+            
+        except Exception as e:
+            raise StorageError(
+                operation="upsert_points",
+                collection=collection,
+                reason=str(e)
+            )
+    
+    def get_collection_info(self, name: str) -> Optional[CollectionInfo]:
+        """Get information about a collection."""
+        if not self._initialized:
+            return None
+        
+        try:
+            return self.client.get_collection(collection_name=name)
+        except Exception as e:
+            logger.error(f"Failed to get collection info: {e}")
+            return None
+    
+    def delete_collection(self, name: str) -> bool:
+        """Delete a collection."""
+        if not self._initialized:
+            raise StorageError(
+                operation="delete_collection",
+                collection=name,
+                reason="Storage not initialized"
+            )
+        
+        try:
+            if not self.collection_exists(name):
+                return False
+            
+            self.client.delete_collection(collection_name=name)
+            logger.info(f"Deleted collection {name}")
+            return True
+            
+        except Exception as e:
+            raise StorageError(
+                operation="delete_collection",
+                collection=name,
+                reason=str(e)
+            )
+    
+    def get_collections(self) -> List[str]:
+        """Get list of all collection names."""
+        if not self._initialized:
+            return []
+        
+        try:
+            collections = self.client.get_collections().collections
+            return [c.name for c in collections]
+        except Exception as e:
+            logger.error(f"Failed to get collections: {e}")
+            return []
+    
+    def count_points(self, collection: str) -> int:
+        """Count points in a collection."""
+        if not self._initialized:
+            return 0
+        
+        try:
+            info = self.get_collection_info(collection)
+            return info.points_count if info else 0
+        except Exception as e:
+            logger.error(f"Failed to count points: {e}")
+            return 0
+    
+    def _generate_point_id(self, string_id: str) -> str:
+        """
+        Generate a valid point ID from a string.
+        
+        Qdrant accepts string IDs directly in newer versions.
+        For older versions, we might need to hash to integer.
+        """
+        # For now, use string IDs directly
+        # If needed, can hash: int(hashlib.md5(string_id.encode()).hexdigest()[:16], 16)
+        return string_id
+    
+    def health_check(self) -> Dict[str, Any]:
+        """Check health of Qdrant connection."""
+        if not self._initialized:
+            return {"healthy": False, "reason": "Not initialized"}
+        
+        try:
+            collections = self.client.get_collections()
+            return {
+                "healthy": True,
+                "url": self.url,
+                "collections_count": len(collections.collections),
+                "total_points": sum(
+                    self.count_points(c.name) 
+                    for c in collections.collections
+                )
+            }
+        except Exception as e:
+            return {"healthy": False, "reason": str(e)}

package/scripts/importer/utils/__init__.py

@@ -0,0 +1,9 @@
+"""Utility functions for the import system."""
+
+from .project_normalizer import ProjectNormalizer
+from .logger import setup_logging
+
+__all__ = [
+    "ProjectNormalizer",
+    "setup_logging"
+]