claude-self-reflect 3.0.0 → 3.0.2

package/scripts/importer/processors/ast_extractor.py

@@ -0,0 +1,197 @@
+"""Extract AST elements from code blocks."""
+
+import ast
+import re
+import logging
+from typing import Dict, Any, Set, List
+
+logger = logging.getLogger(__name__)
+
+
+class ASTExtractor:
+    """
+    Extract Abstract Syntax Tree elements from code.
+    
+    Implements the critical fixes identified in code review:
+    1. More permissive code fence regex
+    2. Python regex fallback for partial code
+    3. Bounded extraction with MAX_AST_ELEMENTS
+    """
+    
+    def __init__(self, max_elements: int = 100):
+        self.max_elements = max_elements
+        
+        # FIX: More permissive code fence regex to handle various formats
+        # Matches: ```python, ```py, ```javascript, ```ts strict, etc.
+        self.code_fence_pattern = re.compile(
+            r'```[^\n]*\n?(.*?)```',
+            re.DOTALL
+        )
+        
+        # Python patterns for fallback extraction
+        self.python_patterns = {
+            'function': re.compile(r'^\s*def\s+([A-Za-z_]\w*)\s*\(', re.MULTILINE),
+            'async_function': re.compile(r'^\s*async\s+def\s+([A-Za-z_]\w*)\s*\(', re.MULTILINE),
+            'class': re.compile(r'^\s*class\s+([A-Za-z_]\w*)\s*[:\(]', re.MULTILINE),
+            'method': re.compile(r'^\s+def\s+([A-Za-z_]\w*)\s*\(self', re.MULTILINE),
+            'static_method': re.compile(r'@staticmethod.*?\n\s*def\s+([A-Za-z_]\w*)\s*\(', re.MULTILINE | re.DOTALL),
+            'class_method': re.compile(r'@classmethod.*?\n\s*def\s+([A-Za-z_]\w*)\s*\(', re.MULTILINE | re.DOTALL)
+        }
+        
+        # JavaScript/TypeScript patterns
+        self.js_patterns = {
+            'function': re.compile(r'function\s+([A-Za-z_$][\w$]*)\s*\('),
+            'arrow': re.compile(r'(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:\([^)]*\)|[A-Za-z_$][\w$]*)\s*=>'),
+            'async_function': re.compile(r'async\s+function\s+([A-Za-z_$][\w$]*)\s*\('),
+            'class': re.compile(r'class\s+([A-Za-z_$][\w$]*)\s*(?:extends\s+[A-Za-z_$][\w$]*)?\s*\{'),
+            'method': re.compile(r'([A-Za-z_$][\w$]*)\s*\([^)]*\)\s*\{'),
+            'export_function': re.compile(r'export\s+(?:default\s+)?(?:async\s+)?function\s+([A-Za-z_$][\w$]*)\s*\('),
+            'export_const': re.compile(r'export\s+const\s+([A-Za-z_$][\w$]*)\s*=')
+        }
+    
+    def extract(self, text: str) -> Dict[str, Any]:
+        """
+        Extract AST elements from text.
+        
+        Returns:
+            Dictionary with ast_elements and code_blocks keys
+        """
+        elements = set()
+        has_code = False
+        
+        # Extract code blocks using permissive regex
+        code_blocks = self.code_fence_pattern.findall(text)
+        
+        for code_block in code_blocks[:10]:  # Limit processing
+            has_code = True
+            
+            # Try to detect language from content
+            if self._looks_like_python(code_block):
+                python_elements = self._extract_python_ast(code_block)
+                elements.update(python_elements)
+            elif self._looks_like_javascript(code_block):
+                js_elements = self._extract_javascript_patterns(code_block)
+                elements.update(js_elements)
+            else:
+                # Try both as fallback
+                elements.update(self._extract_python_ast(code_block))
+                elements.update(self._extract_javascript_patterns(code_block))
+            
+            # FIX: Enforce max elements limit
+            if len(elements) >= self.max_elements:
+                logger.debug(f"Reached max AST elements limit: {self.max_elements}")
+                break
+        
+        # Also check for inline code patterns outside of fences
+        if not has_code:
+            # Look for function/class definitions in plain text
+            elements.update(self._extract_inline_patterns(text))
+        
+        return {
+            "ast_elements": list(elements)[:self.max_elements],
+            "has_code_blocks": has_code
+        }
+    
+    def _extract_python_ast(self, code: str) -> Set[str]:
+        """Extract Python AST elements with fallback to regex."""
+        elements = set()
+        
+        try:
+            # Try proper AST parsing first
+            tree = ast.parse(code)
+            
+            for node in ast.walk(tree):
+                if len(elements) >= self.max_elements:
+                    break
+                
+                if isinstance(node, ast.FunctionDef):
+                    elements.add(f"func:{node.name}")
+                elif isinstance(node, ast.AsyncFunctionDef):
+                    elements.add(f"func:{node.name}")
+                elif isinstance(node, ast.ClassDef):
+                    elements.add(f"class:{node.name}")
+                    # Extract methods
+                    for item in node.body:
+                        if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                            elements.add(f"method:{node.name}.{item.name}")
+                            if len(elements) >= self.max_elements:
+                                break
+                
+        except (SyntaxError, ValueError) as e:
+            # FIX: Python regex fallback for partial code fragments
+            logger.debug(f"AST parsing failed, using regex fallback: {e}")
+            
+            for pattern_type, pattern in self.python_patterns.items():
+                for match in pattern.finditer(code):
+                    if len(elements) >= self.max_elements:
+                        break
+                    
+                    name = match.group(1)
+                    if 'method' in pattern_type:
+                        elements.add(f"method:{name}")
+                    elif 'class' in pattern_type:
+                        elements.add(f"class:{name}")
+                    else:
+                        elements.add(f"func:{name}")
+        
+        return elements
+    
+    def _extract_javascript_patterns(self, code: str) -> Set[str]:
+        """Extract JavaScript/TypeScript patterns."""
+        elements = set()
+        
+        for pattern_type, pattern in self.js_patterns.items():
+            for match in pattern.finditer(code):
+                if len(elements) >= self.max_elements:
+                    break
+                
+                name = match.group(1)
+                if 'class' in pattern_type:
+                    elements.add(f"class:{name}")
+                elif 'method' in pattern_type and name not in ['constructor', 'if', 'for', 'while']:
+                    elements.add(f"method:{name}")
+                else:
+                    elements.add(f"func:{name}")
+        
+        return elements
+    
+    def _extract_inline_patterns(self, text: str) -> Set[str]:
+        """Extract patterns from inline code mentions."""
+        elements = set()
+        
+        # Look for backtick-wrapped function/class names
+        inline_pattern = re.compile(r'`([A-Za-z_][\w]*(?:\.[A-Za-z_][\w]*)*)`')
+        
+        for match in inline_pattern.finditer(text):
+            if len(elements) >= self.max_elements:
+                break
+            
+            name = match.group(1)
+            # Heuristic: if contains dot, likely a method
+            if '.' in name:
+                elements.add(f"method:{name}")
+            # Heuristic: PascalCase likely a class
+            elif name[0].isupper():
+                elements.add(f"class:{name}")
+            # Otherwise assume function
+            else:
+                elements.add(f"func:{name}")
+        
+        return elements
+    
+    def _looks_like_python(self, code: str) -> bool:
+        """Heuristic to detect Python code."""
+        python_indicators = [
+            'def ', 'import ', 'from ', 'class ', 'self.', 'self,',
+            '__init__', '__name__', 'if __name__', 'print(', 'async def'
+        ]
+        return any(indicator in code for indicator in python_indicators)
+    
+    def _looks_like_javascript(self, code: str) -> bool:
+        """Heuristic to detect JavaScript/TypeScript."""
+        js_indicators = [
+            'function ', 'const ', 'let ', 'var ', '=>', 'export ',
+            'import ', 'class ', 'constructor(', 'this.', 'async function',
+            'interface ', 'type ', 'namespace ', 'enum '
+        ]
+        return any(indicator in code for indicator in js_indicators)

package/scripts/importer/processors/chunker.py

@@ -0,0 +1,157 @@
+"""Intelligent chunking for conversations."""
+
+import logging
+from typing import List, Optional
+from pathlib import Path
+
+from ..core import Message, ConversationChunk
+
+logger = logging.getLogger(__name__)
+
+
+class Chunker:
+    """
+    Create optimized chunks from conversation messages.
+    
+    Implements intelligent chunking with overlap for context preservation.
+    """
+    
+    def __init__(self, chunk_size: int = 3000, chunk_overlap: int = 200):
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+    
+    def create_chunks(
+        self, 
+        messages: List[Message],
+        file_path: str
+    ) -> List[ConversationChunk]:
+        """
+        Create chunks from messages.
+        
+        Args:
+            messages: List of messages to chunk
+            file_path: Source file path for metadata
+            
+        Returns:
+            List of conversation chunks
+        """
+        if not messages:
+            return []
+        
+        # Generate conversation ID from file path
+        conversation_id = Path(file_path).stem
+        
+        chunks = []
+        current_chunk_text = []
+        current_chunk_size = 0
+        current_message_indices = []
+        
+        for msg in messages:
+            # Format message for chunk
+            formatted = self._format_message(msg)
+            msg_size = len(formatted)
+            
+            # Check if adding this message would exceed chunk size
+            if current_chunk_size + msg_size > self.chunk_size and current_chunk_text:
+                # Create chunk with current messages
+                chunk = self._create_chunk(
+                    current_chunk_text,
+                    current_message_indices,
+                    len(chunks),
+                    conversation_id,
+                    file_path
+                )
+                chunks.append(chunk)
+                
+                # Start new chunk with overlap
+                overlap_text, overlap_indices = self._get_overlap(
+                    current_chunk_text,
+                    current_message_indices
+                )
+                current_chunk_text = overlap_text
+                current_message_indices = overlap_indices
+                current_chunk_size = sum(len(t) for t in current_chunk_text)
+            
+            # Add message to current chunk
+            current_chunk_text.append(formatted)
+            # Fix: Check for None instead of truthiness to include index 0
+            if msg.message_index is not None:
+                current_message_indices.append(msg.message_index)
+            current_chunk_size += msg_size
+        
+        # Create final chunk
+        if current_chunk_text:
+            chunk = self._create_chunk(
+                current_chunk_text,
+                current_message_indices,
+                len(chunks),
+                conversation_id,
+                file_path
+            )
+            chunks.append(chunk)
+        
+        # Update total chunks count
+        for chunk in chunks:
+            chunk.total_chunks = len(chunks)
+        
+        logger.debug(f"Created {len(chunks)} chunks from {len(messages)} messages")
+        return chunks
+    
+    def _format_message(self, message: Message) -> str:
+        """Format a message for inclusion in chunk."""
+        # Include role for context
+        role_prefix = f"[{message.role.upper()}]: "
+        return role_prefix + message.content
+    
+    def _get_overlap(
+        self,
+        chunk_text: List[str],
+        message_indices: List[int]
+    ) -> tuple[List[str], List[int]]:
+        """Get overlap text and indices for next chunk."""
+        if not chunk_text:
+            return [], []
+        
+        # Calculate how many messages to include in overlap
+        overlap_size = 0
+        overlap_messages = []
+        overlap_indices = []
+        
+        # Work backwards to get overlap
+        for i in range(len(chunk_text) - 1, -1, -1):
+            msg_size = len(chunk_text[i])
+            if overlap_size + msg_size <= self.chunk_overlap:
+                overlap_messages.insert(0, chunk_text[i])
+                if i < len(message_indices):
+                    overlap_indices.insert(0, message_indices[i])
+                overlap_size += msg_size
+            else:
+                break
+        
+        return overlap_messages, overlap_indices
+    
+    def _create_chunk(
+        self,
+        text_parts: List[str],
+        message_indices: List[int],
+        chunk_index: int,
+        conversation_id: str,
+        file_path: str
+    ) -> ConversationChunk:
+        """Create a conversation chunk."""
+        chunk_text = "\n".join(text_parts)
+        
+        chunk = ConversationChunk(
+            text=chunk_text,
+            message_indices=message_indices,
+            chunk_index=chunk_index,
+            total_chunks=0,  # Will be updated later
+            conversation_id=conversation_id
+        )
+        
+        # Add file metadata
+        chunk.add_metadata("file_path", file_path)
+        chunk.add_metadata("chunk_method", "overlap")
+        chunk.add_metadata("chunk_size_chars", len(chunk_text))
+        
+        return chunk

package/scripts/importer/processors/concept_extractor.py

@@ -0,0 +1,109 @@
+"""Extract concepts and keywords from text."""
+
+import re
+import logging
+from typing import Dict, Any, Set, List
+from collections import Counter
+
+logger = logging.getLogger(__name__)
+
+
+class ConceptExtractor:
+    """Extract key concepts and keywords from conversation text."""
+    
+    def __init__(self):
+        # Technical concepts to look for
+        self.tech_patterns = {
+            'languages': re.compile(r'\b(python|javascript|typescript|java|rust|go|c\+\+|ruby|php|swift|kotlin)\b', re.IGNORECASE),
+            'frameworks': re.compile(r'\b(react|vue|angular|django|flask|fastapi|express|spring|rails|laravel)\b', re.IGNORECASE),
+            'databases': re.compile(r'\b(mongodb|postgres|mysql|redis|elasticsearch|dynamodb|sqlite|cassandra)\b', re.IGNORECASE),
+            'cloud': re.compile(r'\b(aws|azure|gcp|docker|kubernetes|serverless|lambda|ec2|s3)\b', re.IGNORECASE),
+            'tools': re.compile(r'\b(git|npm|yarn|webpack|babel|eslint|pytest|jest|vscode|vim)\b', re.IGNORECASE),
+            'concepts': re.compile(r'\b(api|rest|graphql|microservices|ci\/cd|devops|agile|tdd|security|authentication)\b', re.IGNORECASE)
+        }
+        
+        # Common stop words to exclude
+        self.stop_words = {
+            'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
+            'of', 'with', 'by', 'from', 'as', 'is', 'was', 'are', 'were', 'been',
+            'be', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would',
+            'should', 'could', 'may', 'might', 'must', 'can', 'shall', 'need'
+        }
+    
+    def extract(self, text: str) -> Dict[str, Any]:
+        """
+        Extract concepts from text.
+        
+        Returns:
+            Dictionary with concepts list
+        """
+        concepts = set()
+        
+        # Extract technical concepts
+        for category, pattern in self.tech_patterns.items():
+            matches = pattern.findall(text)
+            for match in matches:
+                concepts.add(match.lower())
+        
+        # Extract capitalized terms (likely important)
+        capitalized = re.findall(r'\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\b', text)
+        for term in capitalized[:20]:  # Limit to prevent noise
+            if term.lower() not in self.stop_words and len(term) > 3:
+                concepts.add(term.lower())
+        
+        # Extract terms in backticks (code references)
+        code_terms = re.findall(r'`([^`]+)`', text)
+        for term in code_terms[:20]:
+            # Clean and add if it's a reasonable concept
+            clean_term = term.strip().lower()
+            if len(clean_term) > 2 and len(clean_term) < 50:
+                # Skip if it looks like code
+                if not any(char in clean_term for char in ['{', '}', '(', ')', ';', '=']):
+                    concepts.add(clean_term)
+        
+        # Extract file extensions mentioned
+        extensions = re.findall(r'\b\w+\.(py|js|ts|jsx|tsx|java|go|rs|cpp|c|h|md|json|yaml|yml|xml|html|css|sql)\b', text)
+        for ext in extensions[:10]:
+            concepts.add(f"file:{ext.split('.')[-1]}")
+        
+        # Extract error types
+        errors = re.findall(r'\b(\w+Error|Exception)\b', text)
+        for error in errors[:10]:
+            concepts.add(f"error:{error.lower()}")
+        
+        # Limit total concepts to prevent bloat
+        concept_list = list(concepts)[:50]
+        
+        return {
+            "concepts": concept_list,
+            "concept_count": len(concept_list)
+        }
+    
+    def extract_topics(self, text: str) -> List[str]:
+        """
+        Extract higher-level topics from text.
+        
+        This is a more sophisticated extraction for topic modeling.
+        """
+        topics = []
+        
+        # Check for common development topics
+        topic_indicators = {
+            'debugging': ['debug', 'error', 'bug', 'fix', 'issue', 'problem'],
+            'testing': ['test', 'unit test', 'integration', 'pytest', 'jest', 'coverage'],
+            'deployment': ['deploy', 'production', 'release', 'ci/cd', 'pipeline'],
+            'optimization': ['optimize', 'performance', 'speed', 'efficiency', 'cache'],
+            'security': ['security', 'authentication', 'authorization', 'encryption', 'vulnerability'],
+            'database': ['database', 'sql', 'query', 'schema', 'migration'],
+            'api': ['api', 'endpoint', 'rest', 'graphql', 'webhook'],
+            'frontend': ['ui', 'ux', 'component', 'react', 'vue', 'css', 'style'],
+            'backend': ['server', 'backend', 'api', 'database', 'microservice'],
+            'architecture': ['architecture', 'design pattern', 'structure', 'refactor']
+        }
+        
+        text_lower = text.lower()
+        for topic, indicators in topic_indicators.items():
+            if any(indicator in text_lower for indicator in indicators):
+                topics.append(topic)
+        
+        return topics[:5]  # Return top 5 topics

package/scripts/importer/processors/conversation_parser.py

@@ -0,0 +1,181 @@
+"""Parser for JSONL conversation files."""
+
+import json
+import logging
+from pathlib import Path
+from typing import List, Dict, Any, Optional
+from datetime import datetime
+
+from ..core import Message
+from ..core.exceptions import ParseError
+
+logger = logging.getLogger(__name__)
+
+
+class ConversationParser:
+    """
+    Parse JSONL conversation files into Message objects.
+    
+    Handles various conversation formats from Claude.
+    """
+    
+    def parse_file(self, file_path: Path) -> List[Message]:
+        """
+        Parse a JSONL file into messages.
+        
+        Args:
+            file_path: Path to JSONL file
+            
+        Returns:
+            List of Message objects
+            
+        Raises:
+            ParseError: If parsing fails
+        """
+        messages = []
+        
+        try:
+            with open(file_path, 'r', encoding='utf-8') as f:
+                for line_num, line in enumerate(f, 1):
+                    line = line.strip()
+                    if not line:
+                        continue
+                    
+                    try:
+                        data = json.loads(line)
+                        message = self._parse_message(data, line_num)
+                        if message:
+                            messages.append(message)
+                    except json.JSONDecodeError as e:
+                        logger.warning(f"Skipping invalid JSON at line {line_num}: {e}")
+                        # Don't fail entire file for one bad line
+                        continue
+            
+            if not messages:
+                raise ParseError(
+                    str(file_path),
+                    reason="No valid messages found in file"
+                )
+            
+            # Add message indices
+            for i, msg in enumerate(messages):
+                msg.message_index = i
+            
+            logger.debug(f"Parsed {len(messages)} messages from {file_path}")
+            return messages
+            
+        except FileNotFoundError:
+            raise ParseError(str(file_path), reason="File not found")
+        except Exception as e:
+            if isinstance(e, ParseError):
+                raise
+            raise ParseError(str(file_path), reason=str(e))
+    
+    def _parse_message(self, data: Dict[str, Any], line_num: int) -> Optional[Message]:
+        """
+        Parse a single message from JSON data.
+        
+        Handles multiple conversation formats.
+        """
+        # Format 1: Direct message format
+        if "role" in data and "content" in data:
+            return Message(
+                role=data["role"],
+                content=self._extract_content(data["content"]),
+                timestamp=self._parse_timestamp(data.get("timestamp")),
+                metadata=self._extract_metadata(data)
+            )
+        
+        # Format 2: Nested messages array
+        if "messages" in data and isinstance(data["messages"], list):
+            # Return first message or aggregate
+            messages = []
+            for msg_data in data["messages"]:
+                if isinstance(msg_data, dict) and "role" in msg_data:
+                    msg = Message(
+                        role=msg_data["role"],
+                        content=self._extract_content(msg_data.get("content", "")),
+                        timestamp=self._parse_timestamp(msg_data.get("timestamp")),
+                        metadata=self._extract_metadata(msg_data)
+                    )
+                    messages.append(msg)
+            
+            # For now, return first message
+            # In future, might want to handle differently
+            return messages[0] if messages else None
+        
+        # Format 3: Event-based format
+        if "event" in data and data["event"] == "message":
+            return Message(
+                role=data.get("role", "unknown"),
+                content=self._extract_content(data.get("text", "")),
+                timestamp=self._parse_timestamp(data.get("timestamp")),
+                metadata=self._extract_metadata(data)
+            )
+        
+        # Unknown format
+        logger.debug(f"Unknown message format at line {line_num}")
+        return None
+    
+    def _extract_content(self, content: Any) -> str:
+        """Extract text content from various formats."""
+        if isinstance(content, str):
+            return content
+        
+        if isinstance(content, list):
+            # Handle content array format
+            text_parts = []
+            for item in content:
+                if isinstance(item, dict):
+                    if "text" in item:
+                        text_parts.append(item["text"])
+                    elif "content" in item:
+                        text_parts.append(str(item["content"]))
+                else:
+                    text_parts.append(str(item))
+            return "\n".join(text_parts)
+        
+        if isinstance(content, dict):
+            if "text" in content:
+                return content["text"]
+            elif "content" in content:
+                return str(content["content"])
+        
+        return str(content) if content else ""
+    
+    def _parse_timestamp(self, timestamp: Any) -> Optional[datetime]:
+        """Parse timestamp from various formats."""
+        if not timestamp:
+            return None
+        
+        if isinstance(timestamp, datetime):
+            return timestamp
+        
+        if isinstance(timestamp, (int, float)):
+            # Unix timestamp
+            try:
+                return datetime.fromtimestamp(timestamp)
+            except Exception:
+                return None
+        
+        if isinstance(timestamp, str):
+            # ISO format or other string formats
+            try:
+                return datetime.fromisoformat(timestamp)
+            except Exception:
+                # Try other formats if needed
+                return None
+        
+        return None
+    
+    def _extract_metadata(self, data: Dict[str, Any]) -> Dict[str, Any]:
+        """Extract additional metadata from message data."""
+        # Skip known fields
+        skip_fields = {"role", "content", "text", "timestamp", "message_index"}
+        
+        metadata = {}
+        for key, value in data.items():
+            if key not in skip_fields:
+                metadata[key] = value
+        
+        return metadata