claude-self-reflect 3.0.0 → 3.0.1

package/scripts/importer/processors/chunker.py

@@ -0,0 +1,157 @@
+"""Intelligent chunking for conversations."""
+
+import logging
+from typing import List, Optional
+from pathlib import Path
+
+from ..core import Message, ConversationChunk
+
+logger = logging.getLogger(__name__)
+
+
+class Chunker:
+    """
+    Create optimized chunks from conversation messages.
+    
+    Implements intelligent chunking with overlap for context preservation.
+    """
+    
+    def __init__(self, chunk_size: int = 3000, chunk_overlap: int = 200):
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+    
+    def create_chunks(
+        self, 
+        messages: List[Message],
+        file_path: str
+    ) -> List[ConversationChunk]:
+        """
+        Create chunks from messages.
+        
+        Args:
+            messages: List of messages to chunk
+            file_path: Source file path for metadata
+            
+        Returns:
+            List of conversation chunks
+        """
+        if not messages:
+            return []
+        
+        # Generate conversation ID from file path
+        conversation_id = Path(file_path).stem
+        
+        chunks = []
+        current_chunk_text = []
+        current_chunk_size = 0
+        current_message_indices = []
+        
+        for msg in messages:
+            # Format message for chunk
+            formatted = self._format_message(msg)
+            msg_size = len(formatted)
+            
+            # Check if adding this message would exceed chunk size
+            if current_chunk_size + msg_size > self.chunk_size and current_chunk_text:
+                # Create chunk with current messages
+                chunk = self._create_chunk(
+                    current_chunk_text,
+                    current_message_indices,
+                    len(chunks),
+                    conversation_id,
+                    file_path
+                )
+                chunks.append(chunk)
+                
+                # Start new chunk with overlap
+                overlap_text, overlap_indices = self._get_overlap(
+                    current_chunk_text,
+                    current_message_indices
+                )
+                current_chunk_text = overlap_text
+                current_message_indices = overlap_indices
+                current_chunk_size = sum(len(t) for t in current_chunk_text)
+            
+            # Add message to current chunk
+            current_chunk_text.append(formatted)
+            # Fix: Check for None instead of truthiness to include index 0
+            if msg.message_index is not None:
+                current_message_indices.append(msg.message_index)
+            current_chunk_size += msg_size
+        
+        # Create final chunk
+        if current_chunk_text:
+            chunk = self._create_chunk(
+                current_chunk_text,
+                current_message_indices,
+                len(chunks),
+                conversation_id,
+                file_path
+            )
+            chunks.append(chunk)
+        
+        # Update total chunks count
+        for chunk in chunks:
+            chunk.total_chunks = len(chunks)
+        
+        logger.debug(f"Created {len(chunks)} chunks from {len(messages)} messages")
+        return chunks
+    
+    def _format_message(self, message: Message) -> str:
+        """Format a message for inclusion in chunk."""
+        # Include role for context
+        role_prefix = f"[{message.role.upper()}]: "
+        return role_prefix + message.content
+    
+    def _get_overlap(
+        self,
+        chunk_text: List[str],
+        message_indices: List[int]
+    ) -> tuple[List[str], List[int]]:
+        """Get overlap text and indices for next chunk."""
+        if not chunk_text:
+            return [], []
+        
+        # Calculate how many messages to include in overlap
+        overlap_size = 0
+        overlap_messages = []
+        overlap_indices = []
+        
+        # Work backwards to get overlap
+        for i in range(len(chunk_text) - 1, -1, -1):
+            msg_size = len(chunk_text[i])
+            if overlap_size + msg_size <= self.chunk_overlap:
+                overlap_messages.insert(0, chunk_text[i])
+                if i < len(message_indices):
+                    overlap_indices.insert(0, message_indices[i])
+                overlap_size += msg_size
+            else:
+                break
+        
+        return overlap_messages, overlap_indices
+    
+    def _create_chunk(
+        self,
+        text_parts: List[str],
+        message_indices: List[int],
+        chunk_index: int,
+        conversation_id: str,
+        file_path: str
+    ) -> ConversationChunk:
+        """Create a conversation chunk."""
+        chunk_text = "\n".join(text_parts)
+        
+        chunk = ConversationChunk(
+            text=chunk_text,
+            message_indices=message_indices,
+            chunk_index=chunk_index,
+            total_chunks=0,  # Will be updated later
+            conversation_id=conversation_id
+        )
+        
+        # Add file metadata
+        chunk.add_metadata("file_path", file_path)
+        chunk.add_metadata("chunk_method", "overlap")
+        chunk.add_metadata("chunk_size_chars", len(chunk_text))
+        
+        return chunk

package/scripts/importer/processors/concept_extractor.py

@@ -0,0 +1,109 @@
+"""Extract concepts and keywords from text."""
+
+import re
+import logging
+from typing import Dict, Any, Set, List
+from collections import Counter
+
+logger = logging.getLogger(__name__)
+
+
+class ConceptExtractor:
+    """Extract key concepts and keywords from conversation text."""
+    
+    def __init__(self):
+        # Technical concepts to look for
+        self.tech_patterns = {
+            'languages': re.compile(r'\b(python|javascript|typescript|java|rust|go|c\+\+|ruby|php|swift|kotlin)\b', re.IGNORECASE),
+            'frameworks': re.compile(r'\b(react|vue|angular|django|flask|fastapi|express|spring|rails|laravel)\b', re.IGNORECASE),
+            'databases': re.compile(r'\b(mongodb|postgres|mysql|redis|elasticsearch|dynamodb|sqlite|cassandra)\b', re.IGNORECASE),
+            'cloud': re.compile(r'\b(aws|azure|gcp|docker|kubernetes|serverless|lambda|ec2|s3)\b', re.IGNORECASE),
+            'tools': re.compile(r'\b(git|npm|yarn|webpack|babel|eslint|pytest|jest|vscode|vim)\b', re.IGNORECASE),
+            'concepts': re.compile(r'\b(api|rest|graphql|microservices|ci\/cd|devops|agile|tdd|security|authentication)\b', re.IGNORECASE)
+        }
+        
+        # Common stop words to exclude
+        self.stop_words = {
+            'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
+            'of', 'with', 'by', 'from', 'as', 'is', 'was', 'are', 'were', 'been',
+            'be', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would',
+            'should', 'could', 'may', 'might', 'must', 'can', 'shall', 'need'
+        }
+    
+    def extract(self, text: str) -> Dict[str, Any]:
+        """
+        Extract concepts from text.
+        
+        Returns:
+            Dictionary with concepts list
+        """
+        concepts = set()
+        
+        # Extract technical concepts
+        for category, pattern in self.tech_patterns.items():
+            matches = pattern.findall(text)
+            for match in matches:
+                concepts.add(match.lower())
+        
+        # Extract capitalized terms (likely important)
+        capitalized = re.findall(r'\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\b', text)
+        for term in capitalized[:20]:  # Limit to prevent noise
+            if term.lower() not in self.stop_words and len(term) > 3:
+                concepts.add(term.lower())
+        
+        # Extract terms in backticks (code references)
+        code_terms = re.findall(r'`([^`]+)`', text)
+        for term in code_terms[:20]:
+            # Clean and add if it's a reasonable concept
+            clean_term = term.strip().lower()
+            if len(clean_term) > 2 and len(clean_term) < 50:
+                # Skip if it looks like code
+                if not any(char in clean_term for char in ['{', '}', '(', ')', ';', '=']):
+                    concepts.add(clean_term)
+        
+        # Extract file extensions mentioned
+        extensions = re.findall(r'\b\w+\.(py|js|ts|jsx|tsx|java|go|rs|cpp|c|h|md|json|yaml|yml|xml|html|css|sql)\b', text)
+        for ext in extensions[:10]:
+            concepts.add(f"file:{ext.split('.')[-1]}")
+        
+        # Extract error types
+        errors = re.findall(r'\b(\w+Error|Exception)\b', text)
+        for error in errors[:10]:
+            concepts.add(f"error:{error.lower()}")
+        
+        # Limit total concepts to prevent bloat
+        concept_list = list(concepts)[:50]
+        
+        return {
+            "concepts": concept_list,
+            "concept_count": len(concept_list)
+        }
+    
+    def extract_topics(self, text: str) -> List[str]:
+        """
+        Extract higher-level topics from text.
+        
+        This is a more sophisticated extraction for topic modeling.
+        """
+        topics = []
+        
+        # Check for common development topics
+        topic_indicators = {
+            'debugging': ['debug', 'error', 'bug', 'fix', 'issue', 'problem'],
+            'testing': ['test', 'unit test', 'integration', 'pytest', 'jest', 'coverage'],
+            'deployment': ['deploy', 'production', 'release', 'ci/cd', 'pipeline'],
+            'optimization': ['optimize', 'performance', 'speed', 'efficiency', 'cache'],
+            'security': ['security', 'authentication', 'authorization', 'encryption', 'vulnerability'],
+            'database': ['database', 'sql', 'query', 'schema', 'migration'],
+            'api': ['api', 'endpoint', 'rest', 'graphql', 'webhook'],
+            'frontend': ['ui', 'ux', 'component', 'react', 'vue', 'css', 'style'],
+            'backend': ['server', 'backend', 'api', 'database', 'microservice'],
+            'architecture': ['architecture', 'design pattern', 'structure', 'refactor']
+        }
+        
+        text_lower = text.lower()
+        for topic, indicators in topic_indicators.items():
+            if any(indicator in text_lower for indicator in indicators):
+                topics.append(topic)
+        
+        return topics[:5]  # Return top 5 topics

package/scripts/importer/processors/conversation_parser.py

@@ -0,0 +1,181 @@
+"""Parser for JSONL conversation files."""
+
+import json
+import logging
+from pathlib import Path
+from typing import List, Dict, Any, Optional
+from datetime import datetime
+
+from ..core import Message
+from ..core.exceptions import ParseError
+
+logger = logging.getLogger(__name__)
+
+
+class ConversationParser:
+    """
+    Parse JSONL conversation files into Message objects.
+    
+    Handles various conversation formats from Claude.
+    """
+    
+    def parse_file(self, file_path: Path) -> List[Message]:
+        """
+        Parse a JSONL file into messages.
+        
+        Args:
+            file_path: Path to JSONL file
+            
+        Returns:
+            List of Message objects
+            
+        Raises:
+            ParseError: If parsing fails
+        """
+        messages = []
+        
+        try:
+            with open(file_path, 'r', encoding='utf-8') as f:
+                for line_num, line in enumerate(f, 1):
+                    line = line.strip()
+                    if not line:
+                        continue
+                    
+                    try:
+                        data = json.loads(line)
+                        message = self._parse_message(data, line_num)
+                        if message:
+                            messages.append(message)
+                    except json.JSONDecodeError as e:
+                        logger.warning(f"Skipping invalid JSON at line {line_num}: {e}")
+                        # Don't fail entire file for one bad line
+                        continue
+            
+            if not messages:
+                raise ParseError(
+                    str(file_path),
+                    reason="No valid messages found in file"
+                )
+            
+            # Add message indices
+            for i, msg in enumerate(messages):
+                msg.message_index = i
+            
+            logger.debug(f"Parsed {len(messages)} messages from {file_path}")
+            return messages
+            
+        except FileNotFoundError:
+            raise ParseError(str(file_path), reason="File not found")
+        except Exception as e:
+            if isinstance(e, ParseError):
+                raise
+            raise ParseError(str(file_path), reason=str(e))
+    
+    def _parse_message(self, data: Dict[str, Any], line_num: int) -> Optional[Message]:
+        """
+        Parse a single message from JSON data.
+        
+        Handles multiple conversation formats.
+        """
+        # Format 1: Direct message format
+        if "role" in data and "content" in data:
+            return Message(
+                role=data["role"],
+                content=self._extract_content(data["content"]),
+                timestamp=self._parse_timestamp(data.get("timestamp")),
+                metadata=self._extract_metadata(data)
+            )
+        
+        # Format 2: Nested messages array
+        if "messages" in data and isinstance(data["messages"], list):
+            # Return first message or aggregate
+            messages = []
+            for msg_data in data["messages"]:
+                if isinstance(msg_data, dict) and "role" in msg_data:
+                    msg = Message(
+                        role=msg_data["role"],
+                        content=self._extract_content(msg_data.get("content", "")),
+                        timestamp=self._parse_timestamp(msg_data.get("timestamp")),
+                        metadata=self._extract_metadata(msg_data)
+                    )
+                    messages.append(msg)
+            
+            # For now, return first message
+            # In future, might want to handle differently
+            return messages[0] if messages else None
+        
+        # Format 3: Event-based format
+        if "event" in data and data["event"] == "message":
+            return Message(
+                role=data.get("role", "unknown"),
+                content=self._extract_content(data.get("text", "")),
+                timestamp=self._parse_timestamp(data.get("timestamp")),
+                metadata=self._extract_metadata(data)
+            )
+        
+        # Unknown format
+        logger.debug(f"Unknown message format at line {line_num}")
+        return None
+    
+    def _extract_content(self, content: Any) -> str:
+        """Extract text content from various formats."""
+        if isinstance(content, str):
+            return content
+        
+        if isinstance(content, list):
+            # Handle content array format
+            text_parts = []
+            for item in content:
+                if isinstance(item, dict):
+                    if "text" in item:
+                        text_parts.append(item["text"])
+                    elif "content" in item:
+                        text_parts.append(str(item["content"]))
+                else:
+                    text_parts.append(str(item))
+            return "\n".join(text_parts)
+        
+        if isinstance(content, dict):
+            if "text" in content:
+                return content["text"]
+            elif "content" in content:
+                return str(content["content"])
+        
+        return str(content) if content else ""
+    
+    def _parse_timestamp(self, timestamp: Any) -> Optional[datetime]:
+        """Parse timestamp from various formats."""
+        if not timestamp:
+            return None
+        
+        if isinstance(timestamp, datetime):
+            return timestamp
+        
+        if isinstance(timestamp, (int, float)):
+            # Unix timestamp
+            try:
+                return datetime.fromtimestamp(timestamp)
+            except Exception:
+                return None
+        
+        if isinstance(timestamp, str):
+            # ISO format or other string formats
+            try:
+                return datetime.fromisoformat(timestamp)
+            except Exception:
+                # Try other formats if needed
+                return None
+        
+        return None
+    
+    def _extract_metadata(self, data: Dict[str, Any]) -> Dict[str, Any]:
+        """Extract additional metadata from message data."""
+        # Skip known fields
+        skip_fields = {"role", "content", "text", "timestamp", "message_index"}
+        
+        metadata = {}
+        for key, value in data.items():
+            if key not in skip_fields:
+                metadata[key] = value
+        
+        return metadata

package/scripts/importer/processors/tool_extractor.py

@@ -0,0 +1,165 @@
+"""Extract tool usage and file references from conversations."""
+
+import re
+import logging
+from typing import Dict, Any, Set, List
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+class ToolUsageExtractor:
+    """Extract files analyzed, edited, and tools used from conversations."""
+    
+    def __init__(self):
+        # Patterns for file operations
+        self.file_patterns = {
+            'analyzed': [
+                re.compile(r'(?:reading|analyzing|examining|looking at|checking)\s+(?:file\s+)?([/\w\-\.]+\.\w+)', re.IGNORECASE),
+                re.compile(r'(?:Read|read)\s+([/\w\-\.]+\.\w+)', re.IGNORECASE),
+                re.compile(r'(?:in|from)\s+file\s+([/\w\-\.]+\.\w+)', re.IGNORECASE)
+            ],
+            'edited': [
+                re.compile(r'(?:editing|modifying|updating|changing|writing to)\s+(?:file\s+)?([/\w\-\.]+\.\w+)', re.IGNORECASE),
+                re.compile(r'(?:Edit|Write)\s+([/\w\-\.]+\.\w+)', re.IGNORECASE),
+                re.compile(r'(?:changes to|modified|updated)\s+([/\w\-\.]+\.\w+)', re.IGNORECASE)
+            ],
+            'created': [
+                re.compile(r'(?:creating|created|new file)\s+([/\w\-\.]+\.\w+)', re.IGNORECASE),
+                re.compile(r'(?:Write|Create)\s+new\s+file\s+([/\w\-\.]+\.\w+)', re.IGNORECASE)
+            ]
+        }
+        
+        # Tool patterns
+        self.tool_patterns = [
+            re.compile(r'(?:using|running|executing)\s+(\w+)\s+(?:tool|command)', re.IGNORECASE),
+            re.compile(r'(?:Tool:|Command:)\s*(\w+)', re.IGNORECASE),
+            re.compile(r'```(?:bash|shell|sh)\n([a-z]+)', re.IGNORECASE),
+            re.compile(r'\$\s+([a-z]+)\s+', re.IGNORECASE),  # Command line
+            re.compile(r'(?:npm|yarn|pip|cargo|go|cargo)\s+([\w\-]+)', re.IGNORECASE)
+        ]
+        
+        # MCP tool pattern
+        self.mcp_pattern = re.compile(r'mcp__([a-zA-Z0-9\-_]+)__([a-zA-Z0-9\-_]+)')
+    
+    def extract(self, text: str) -> Dict[str, Any]:
+        """
+        Extract tool usage information from text.
+        
+        Returns:
+            Dictionary with files_analyzed, files_edited, tools_used
+        """
+        files_analyzed = set()
+        files_edited = set()
+        files_created = set()
+        tools_used = set()
+        
+        # Extract file operations
+        for pattern in self.file_patterns['analyzed']:
+            matches = pattern.findall(text)
+            for match in matches:
+                file_path = self._normalize_file_path(match)
+                if file_path:
+                    files_analyzed.add(file_path)
+        
+        for pattern in self.file_patterns['edited']:
+            matches = pattern.findall(text)
+            for match in matches:
+                file_path = self._normalize_file_path(match)
+                if file_path:
+                    files_edited.add(file_path)
+        
+        for pattern in self.file_patterns['created']:
+            matches = pattern.findall(text)
+            for match in matches:
+                file_path = self._normalize_file_path(match)
+                if file_path:
+                    files_created.add(file_path)
+        
+        # Extract tools
+        for pattern in self.tool_patterns:
+            matches = pattern.findall(text)
+            for match in matches:
+                tool = match.lower().strip()
+                if self._is_valid_tool(tool):
+                    tools_used.add(tool)
+        
+        # Extract MCP tools specifically
+        mcp_matches = self.mcp_pattern.findall(text)
+        for server, tool in mcp_matches:
+            tools_used.add(f"mcp:{server}:{tool}")
+        
+        # Look for common CLI tools
+        common_tools = [
+            'git', 'npm', 'yarn', 'pip', 'python', 'node', 'docker',
+            'kubectl', 'aws', 'gcloud', 'az', 'terraform', 'ansible',
+            'make', 'gradle', 'maven', 'cargo', 'go', 'rustc'
+        ]
+        for tool in common_tools:
+            if re.search(rf'\b{tool}\b', text, re.IGNORECASE):
+                tools_used.add(tool)
+        
+        # Combine all files for backward compatibility
+        all_files = files_analyzed | files_edited | files_created
+        
+        return {
+            "files_analyzed": list(files_analyzed)[:50],
+            "files_edited": list(files_edited)[:50],
+            "files_created": list(files_created)[:50],
+            "files": list(all_files)[:50],  # Legacy field
+            "tools_used": list(tools_used)[:30],
+            "file_count": len(all_files),
+            "tool_count": len(tools_used)
+        }
+    
+    def _normalize_file_path(self, path: str) -> str:
+        """Normalize and validate file path."""
+        # Remove quotes and whitespace
+        path = path.strip('\'"` \n')
+        
+        # Skip if too short or too long
+        if len(path) < 3 or len(path) > 200:
+            return ""
+        
+        # Must have an extension
+        if '.' not in path:
+            return ""
+        
+        # Extract just the filename if it's a full path
+        if '/' in path:
+            # Get the last component
+            path = path.split('/')[-1]
+        
+        # Validate extension
+        valid_extensions = {
+            'py', 'js', 'ts', 'jsx', 'tsx', 'java', 'go', 'rs', 'cpp', 'c', 'h',
+            'md', 'txt', 'json', 'yaml', 'yml', 'xml', 'html', 'css', 'scss',
+            'sql', 'sh', 'bash', 'dockerfile', 'makefile', 'toml', 'ini', 'cfg'
+        }
+        
+        ext = path.split('.')[-1].lower()
+        if ext not in valid_extensions:
+            return ""
+        
+        return path
+    
+    def _is_valid_tool(self, tool: str) -> bool:
+        """Check if a string is a valid tool name."""
+        # Skip common words
+        skip_words = {
+            'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
+            'this', 'that', 'with', 'from', 'as', 'is', 'was', 'are', 'were'
+        }
+        
+        if tool in skip_words:
+            return False
+        
+        # Must be alphanumeric with possible hyphens/underscores
+        if not re.match(r'^[a-z0-9\-_]+$', tool):
+            return False
+        
+        # Reasonable length
+        if len(tool) < 2 or len(tool) > 30:
+            return False
+        
+        return True

package/scripts/importer/state/__init__.py

@@ -0,0 +1,5 @@
+"""State management for import tracking."""
+
+from .state_manager import StateManager
+
+__all__ = ["StateManager"]