claude-self-reflect 5.0.2 → 5.0.5

package/scripts/import_strategies.py

@@ -0,0 +1,344 @@
+"""
+Import strategies using Strategy pattern to reduce complexity of stream_import_file.
+"""
+
+import json
+import gc
+import os
+import logging
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Dict, Any, List, Optional, Generator
+from datetime import datetime
+
+from message_processors import MessageProcessorFactory
+
+logger = logging.getLogger(__name__)
+
+
+class ImportStrategy(ABC):
+    """Abstract base class for import strategies."""
+
+    @abstractmethod
+    def import_file(self, jsonl_file: Path, collection_name: str, project_path: Path) -> int:
+        """Import a JSONL file using the specific strategy."""
+        pass
+
+
+class ChunkBuffer:
+    """Manages buffering and processing of message chunks."""
+
+    def __init__(self, max_size: int = 50):
+        self.buffer: List[Dict[str, Any]] = []
+        self.max_size = max_size
+        self.current_index = 0
+        # Add memory limit for message content
+        self.max_content_length = int(os.getenv('MAX_MESSAGE_CONTENT_LENGTH', '5000'))
+
+    def add(self, message: Dict[str, Any]) -> bool:
+        """Add a message to the buffer. Returns True if buffer is full."""
+        # Truncate long content to prevent memory issues
+        if 'content' in message and len(message['content']) > self.max_content_length:
+            message = message.copy()
+            message['content'] = message['content'][:self.max_content_length] + '...[truncated]'
+        self.buffer.append(message)
+        return len(self.buffer) >= self.max_size
+
+    def get_and_clear(self) -> List[Dict[str, Any]]:
+        """Get buffer contents and clear it."""
+        contents = self.buffer.copy()
+        self.buffer.clear()
+        return contents
+
+    def has_content(self) -> bool:
+        """Check if buffer has any content."""
+        return len(self.buffer) > 0
+
+
+class MessageStreamReader:
+    """Handles reading and parsing messages from JSONL files."""
+
+    def __init__(self):
+        self.processor_factory = MessageProcessorFactory()
+        self.current_message_index = 0
+
+    def read_messages(self, file_path: Path) -> Generator[Dict[str, Any], None, None]:
+        """Generator that yields processed messages from a JSONL file."""
+        self.current_message_index = 0
+
+        with open(file_path, 'r', encoding='utf-8') as f:
+            for line_num, line in enumerate(f, 1):
+                line = line.strip()
+                if not line:
+                    continue
+
+                message = self._parse_line(line, line_num)
+                if message:
+                    yield message
+
+    def _parse_line(self, line: str, line_num: int) -> Optional[Dict[str, Any]]:
+        """Parse a single line and extract message if present."""
+        try:
+            data = json.loads(line)
+
+            # Skip summary lines
+            if data.get('type') == 'summary':
+                return None
+
+            # Handle message entries
+            if 'message' in data and data['message']:
+                return self._process_message(data['message'])
+
+            # Handle top-level tool entries
+            entry_type = data.get('type')
+            if entry_type in ('tool_result', 'tool_use'):
+                return self._process_tool_entry(data, entry_type)
+
+        except json.JSONDecodeError:
+            logger.debug(f"Skipping invalid JSON at line {line_num}")
+        except (KeyError, TypeError, ValueError) as e:
+            logger.debug(f"Error processing data at line {line_num}: {e}")
+        except Exception as e:
+            logger.warning(f"Unexpected error at line {line_num}: {e}")
+
+        return None
+
+    def _process_message(self, message: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+        """Process a message entry."""
+        role = message.get('role')
+        content = message.get('content')
+
+        if not role or not content:
+            return None
+
+        # Process content
+        text_content = self._extract_text_content(content)
+
+        if not text_content:
+            return None
+
+        # Track message index for user/assistant messages
+        if role in ['user', 'assistant']:
+            message_idx = self.current_message_index
+            self.current_message_index += 1
+        else:
+            message_idx = 0
+
+        return {
+            'role': role,
+            'content': text_content,
+            'message_index': message_idx
+        }
+
+    def _extract_text_content(self, content: Any) -> str:
+        """Extract text content from various content formats."""
+        if isinstance(content, str):
+            return content
+
+        if isinstance(content, list):
+            text_parts = []
+            for item in content:
+                if isinstance(item, dict):
+                    text = self._process_content_item(item)
+                    if text:
+                        text_parts.append(text)
+                elif isinstance(item, str):
+                    text_parts.append(item)
+            return '\n'.join(text_parts)
+
+        return ''
+
+    def _process_content_item(self, item: Dict[str, Any]) -> Optional[str]:
+        """Process a single content item."""
+        item_type = item.get('type', '')
+
+        if item_type == 'text':
+            return item.get('text', '')
+        elif item_type == 'thinking':
+            thinking_content = item.get('thinking', '')
+            return f"[Thinking] {thinking_content[:1000]}" if thinking_content else None
+        elif item_type == 'tool_use':
+            tool_name = item.get('name', 'unknown')
+            tool_input = str(item.get('input', ''))[:500]
+            return f"[Tool: {tool_name}] {tool_input}"
+        elif item_type == 'tool_result':
+            result_content = str(item.get('content', ''))[:1000]
+            return f"[Result] {result_content}"
+
+        return None
+
+    def _process_tool_entry(self, data: Dict[str, Any], entry_type: str) -> Optional[Dict[str, Any]]:
+        """Process a top-level tool entry."""
+        text_parts = []
+
+        if entry_type == 'tool_use':
+            tool_name = data.get('name', 'unknown')
+            tool_input = str(data.get('input', ''))[:500]
+            text_parts.append(f"[Tool: {tool_name}] {tool_input}")
+
+        elif entry_type == 'tool_result':
+            result_content = self._extract_tool_result(data)
+            text_parts.append(f"[Result] {result_content[:1000]}")
+
+        content = "\n".join(text_parts)
+        if not content:
+            return None
+
+        message_idx = self.current_message_index
+        self.current_message_index += 1
+
+        return {
+            'role': entry_type,
+            'content': content,
+            'message_index': message_idx
+        }
+
+    def _extract_tool_result(self, data: Dict[str, Any]) -> str:
+        """Extract result content from tool result data."""
+        result_content = data.get('content')
+
+        if isinstance(result_content, list):
+            flat = []
+            for item in result_content:
+                if isinstance(item, dict) and item.get('type') == 'text':
+                    flat.append(item.get('text', ''))
+                elif isinstance(item, str):
+                    flat.append(item)
+            result_content = "\n".join(flat)
+
+        if not result_content:
+            result_content = data.get('result', '')
+
+        return str(result_content)
+
+
+class StreamImportStrategy(ImportStrategy):
+    """
+    Strategy for streaming import with chunked processing.
+    This is the main refactored implementation.
+    """
+
+    def __init__(self, client, process_chunk_fn, state_manager, max_chunk_size: int = 50,
+                 cleanup_tolerance: int = None):
+        self.client = client
+        self.process_chunk_fn = process_chunk_fn
+        self.state_manager = state_manager
+        self.max_chunk_size = max_chunk_size
+        # Make cleanup tolerance configurable via environment variable
+        self.cleanup_tolerance = cleanup_tolerance or int(os.getenv('CLEANUP_TOLERANCE', '5'))
+        self.stream_reader = MessageStreamReader()
+
+    def import_file(self, jsonl_file: Path, collection_name: str, project_path: Path) -> int:
+        """Import a JSONL file using streaming strategy."""
+        logger.info(f"Streaming import of {jsonl_file.name}")
+
+        conversation_id = jsonl_file.stem
+
+        # Extract metadata first (lightweight)
+        from metadata_extractor import MetadataExtractor
+        extractor = MetadataExtractor()
+        metadata, created_at, total_messages = extractor.extract_metadata_from_file(str(jsonl_file))
+
+        # Initialize chunk processing
+        chunk_buffer = ChunkBuffer(self.max_chunk_size)
+        chunk_index = 0
+        total_chunks = 0
+
+        try:
+            # Stream and process messages
+            for message in self.stream_reader.read_messages(jsonl_file):
+                if chunk_buffer.add(message):
+                    # Buffer is full, process chunk
+                    chunks = self._process_buffer(
+                        chunk_buffer, chunk_index, conversation_id,
+                        created_at, metadata, collection_name, project_path, total_messages
+                    )
+                    total_chunks += chunks
+                    chunk_index += 1
+
+                    # Force garbage collection after each chunk
+                    gc.collect()
+
+                    # Log progress
+                    if chunk_index % 10 == 0:
+                        logger.info(f"Processed {chunk_index} chunks from {jsonl_file.name}")
+
+            # Process remaining messages
+            if chunk_buffer.has_content():
+                chunks = self._process_buffer(
+                    chunk_buffer, chunk_index, conversation_id,
+                    created_at, metadata, collection_name, project_path, total_messages
+                )
+                total_chunks += chunks
+
+            # Clean up old points after successful import
+            if total_chunks > 0:
+                self._cleanup_old_points(conversation_id, collection_name, total_chunks)
+
+            logger.info(f"Imported {total_chunks} chunks from {jsonl_file.name}")
+            return total_chunks
+
+        except (IOError, OSError) as e:
+            logger.error(f"Failed to read file {jsonl_file}: {e}")
+            self._mark_failed(jsonl_file, str(e))
+            return 0
+        except json.JSONDecodeError as e:
+            logger.error(f"Invalid JSON in {jsonl_file}: {e}")
+            self._mark_failed(jsonl_file, str(e))
+            return 0
+        except Exception as e:
+            logger.error(f"Unexpected error importing {jsonl_file}: {e}")
+            self._mark_failed(jsonl_file, str(e))
+            return 0
+
+    def _process_buffer(self, chunk_buffer: ChunkBuffer, chunk_index: int,
+                        conversation_id: str, created_at: str, metadata: Dict[str, Any],
+                        collection_name: str, project_path: Path, total_messages: int) -> int:
+        """Process a buffer of messages and return number of chunks created."""
+        messages = chunk_buffer.get_and_clear()
+        return self.process_chunk_fn(
+            messages, chunk_index, conversation_id,
+            created_at, metadata, collection_name, project_path, total_messages
+        )
+
+    def _cleanup_old_points(self, conversation_id: str, collection_name: str, total_chunks: int):
+        """Clean up old points after successful import."""
+        try:
+            from qdrant_client.models import Filter, FieldCondition, MatchValue
+
+            # Count old points using count API
+            old_count_filter = Filter(
+                must=[FieldCondition(key="conversation_id", match=MatchValue(value=conversation_id))]
+            )
+
+            # Use count API to get actual count
+            old_count = self.client.count(
+                collection_name=collection_name,
+                count_filter=old_count_filter,
+                exact=True
+            ).count
+
+            if old_count > total_chunks + self.cleanup_tolerance:
+                # Use filter parameter for delete
+                self.client.delete(
+                    collection_name=collection_name,
+                    points_selector=Filter(
+                        must=[FieldCondition(key="conversation_id", match=MatchValue(value=conversation_id))]
+                    ),
+                    wait=True
+                )
+                logger.info(f"Deleted {old_count - total_chunks} old points for conversation {conversation_id}")
+
+        except ImportError as e:
+            logger.debug(f"Qdrant client import error: {e}")
+        except Exception as e:
+            logger.warning(f"Could not clean up old points for {conversation_id}: {e}")
+
+    def _mark_failed(self, jsonl_file: Path, error: str):
+        """Mark a file as failed in state manager."""
+        try:
+            self.state_manager.mark_file_failed(str(jsonl_file), error)
+        except AttributeError as e:
+            logger.debug(f"State manager method not available: {e}")
+        except Exception as e:
+            logger.warning(f"Unexpected error marking file as failed: {e}")

package/scripts/message_processors.py

@@ -0,0 +1,248 @@
+"""
+Message processor classes for handling different message types in JSONL import.
+Refactored from extract_metadata_single_pass to reduce complexity.
+"""
+
+import re
+import ast
+import logging
+from abc import ABC, abstractmethod
+from typing import Dict, Any, List, Set, Optional
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+# Constants for metadata limits (can be overridden via environment variables)
+import os
+
+MAX_CONCEPTS = int(os.getenv("MAX_CONCEPTS", "10"))
+MAX_AST_ELEMENTS = int(os.getenv("MAX_AST_ELEMENTS", "30"))
+MAX_CODE_BLOCKS = int(os.getenv("MAX_CODE_BLOCKS", "5"))
+MAX_ELEMENTS_PER_BLOCK = int(os.getenv("MAX_ELEMENTS_PER_BLOCK", "10"))
+MAX_FILES_ANALYZED = int(os.getenv("MAX_FILES_ANALYZED", "20"))
+MAX_FILES_EDITED = int(os.getenv("MAX_FILES_EDITED", "20"))
+MAX_TOOLS_USED = int(os.getenv("MAX_TOOLS_USED", "15"))
+MAX_CONCEPT_MESSAGES = int(os.getenv("MAX_CONCEPT_MESSAGES", "50"))
+
+
+class MessageProcessor(ABC):
+    """Abstract base class for message processing."""
+
+    @abstractmethod
+    def process(self, item: Any, metadata: Dict[str, Any]) -> Optional[str]:
+        """Process a message item and update metadata."""
+        pass
+
+
+class TextMessageProcessor(MessageProcessor):
+    """Process text messages and extract code blocks."""
+
+    def process(self, item: Dict[str, Any], metadata: Dict[str, Any]) -> Optional[str]:
+        """Process text content and extract code blocks with AST elements."""
+        if item.get('type') != 'text':
+            return None
+
+        text_content = item.get('text', '')
+
+        # Check for code blocks
+        if '```' in text_content:
+            metadata['has_code_blocks'] = True
+            self._extract_code_ast_elements(text_content, metadata)
+
+        return text_content
+
+    def _extract_code_ast_elements(self, text: str, metadata: Dict[str, Any]):
+        """Extract AST elements from code blocks in text."""
+        if 'ast_elements' not in metadata:
+            metadata['ast_elements'] = []
+
+        if len(metadata['ast_elements']) >= MAX_AST_ELEMENTS:
+            return
+
+        # More permissive regex to handle various fence formats
+        code_blocks = re.findall(r'```[^`\n]*\n?(.*?)```', text, re.DOTALL)
+
+        for code_block in code_blocks[:MAX_CODE_BLOCKS]:
+            if len(metadata['ast_elements']) >= MAX_AST_ELEMENTS:
+                break
+
+            ast_elems = extract_ast_elements(code_block)
+            for elem in list(ast_elems)[:MAX_ELEMENTS_PER_BLOCK]:
+                if elem not in metadata['ast_elements'] and len(metadata['ast_elements']) < MAX_AST_ELEMENTS:
+                    metadata['ast_elements'].append(elem)
+
+
+class ThinkingMessageProcessor(MessageProcessor):
+    """Process thinking messages."""
+
+    def process(self, item: Dict[str, Any], metadata: Dict[str, Any]) -> Optional[str]:
+        """Process thinking content."""
+        if item.get('type') != 'thinking':
+            return None
+
+        return item.get('thinking', '')
+
+
+class ToolMessageProcessor(MessageProcessor):
+    """Process tool use messages and extract file references."""
+
+    def process(self, item: Dict[str, Any], metadata: Dict[str, Any]) -> Optional[str]:
+        """Process tool use and extract file references."""
+        if item.get('type') != 'tool_use':
+            return None
+
+        tool_name = item.get('name', '')
+
+        # Track tool usage
+        if 'tools_used' not in metadata:
+            metadata['tools_used'] = []
+
+        if tool_name and tool_name not in metadata['tools_used']:
+            if len(metadata['tools_used']) < MAX_TOOLS_USED:
+                metadata['tools_used'].append(tool_name)
+
+        # Extract file references
+        if 'input' in item:
+            self._extract_file_references(item['input'], tool_name, metadata)
+
+        # Return tool use as text
+        tool_input = str(item.get('input', ''))[:500]
+        return f"[Tool: {tool_name}] {tool_input}"
+
+    def _extract_file_references(self, input_data: Any, tool_name: str, metadata: Dict[str, Any]):
+        """Extract file references from tool input."""
+        if not isinstance(input_data, dict):
+            return
+
+        # Initialize metadata lists if not present
+        if 'files_edited' not in metadata:
+            metadata['files_edited'] = []
+        if 'files_analyzed' not in metadata:
+            metadata['files_analyzed'] = []
+
+        is_edit = tool_name in ['Edit', 'Write', 'MultiEdit', 'NotebookEdit']
+
+        # Check file_path field
+        if 'file_path' in input_data:
+            file_ref = input_data['file_path']
+            if is_edit:
+                if file_ref not in metadata['files_edited'] and len(metadata['files_edited']) < MAX_FILES_EDITED:
+                    metadata['files_edited'].append(file_ref)
+            else:
+                if file_ref not in metadata['files_analyzed'] and len(metadata['files_analyzed']) < MAX_FILES_ANALYZED:
+                    metadata['files_analyzed'].append(file_ref)
+
+        # Check path field (for non-edit tools)
+        if 'path' in input_data and not is_edit:
+            file_ref = input_data['path']
+            if file_ref not in metadata['files_analyzed'] and len(metadata['files_analyzed']) < MAX_FILES_ANALYZED:
+                metadata['files_analyzed'].append(file_ref)
+
+
+class ToolResultProcessor(MessageProcessor):
+    """Process tool result messages."""
+
+    def process(self, item: Any, metadata: Dict[str, Any]) -> Optional[str]:
+        """Process tool results."""
+        # Handle both dict items and top-level tool results
+        if isinstance(item, dict):
+            if item.get('type') == 'tool_result':
+                result_content = str(item.get('content', ''))[:1000]
+                return f"[Result] {result_content}"
+            elif item.get('type') == 'tool_use':
+                # Already handled by ToolMessageProcessor
+                return None
+
+        return None
+
+
+class MessageProcessorFactory:
+    """Factory for creating appropriate message processors."""
+
+    def __init__(self):
+        self.processors = {
+            'text': TextMessageProcessor(),
+            'thinking': ThinkingMessageProcessor(),
+            'tool_use': ToolMessageProcessor(),
+            'tool_result': ToolResultProcessor()
+        }
+
+    def get_processor(self, message_type: str) -> Optional[MessageProcessor]:
+        """Get the appropriate processor for a message type."""
+        return self.processors.get(message_type)
+
+    def process_content(self, content: Any, metadata: Dict[str, Any]) -> str:
+        """Process content of various types and return text representation."""
+        text_parts = []
+
+        if isinstance(content, list):
+            for item in content:
+                if isinstance(item, dict):
+                    item_type = item.get('type', '')
+                    processor = self.get_processor(item_type)
+                    if processor:
+                        text = processor.process(item, metadata)
+                        if text:
+                            text_parts.append(text)
+                elif isinstance(item, str):
+                    text_parts.append(item)
+        elif isinstance(content, str):
+            text_parts.append(content)
+
+        return '\n'.join(text_parts)
+
+
+def extract_ast_elements(code_text: str) -> Set[str]:
+    """Extract AST elements from Python code."""
+    elements = set()
+
+    try:
+        tree = ast.parse(code_text)
+        for node in ast.walk(tree):
+            if isinstance(node, ast.FunctionDef):
+                elements.add(f"func:{node.name}")
+            elif isinstance(node, ast.ClassDef):
+                elements.add(f"class:{node.name}")
+            elif isinstance(node, ast.Import):
+                for alias in node.names:
+                    elements.add(f"import:{alias.name}")
+            elif isinstance(node, ast.ImportFrom):
+                module = node.module or ''
+                for alias in node.names:
+                    elements.add(f"from:{module}.{alias.name}")
+    except (SyntaxError, ValueError):
+        # Not Python code or invalid syntax
+        pass
+
+    return elements
+
+
+def extract_concepts(text: str) -> List[str]:
+    """Extract key concepts from text using simple heuristics."""
+    concepts = []
+
+    # Common programming concepts
+    concept_patterns = [
+        (r'\b(async|await|promise|future)\b', 'async-programming'),
+        (r'\b(test|spec|jest|pytest|unittest)\b', 'testing'),
+        (r'\b(docker|container|kubernetes|k8s)\b', 'containerization'),
+        (r'\b(api|rest|graphql|endpoint)\b', 'api-development'),
+        (r'\b(react|vue|angular|svelte)\b', 'frontend-framework'),
+        (r'\b(database|sql|postgres|mysql|mongodb)\b', 'database'),
+        (r'\b(auth|authentication|oauth|jwt)\b', 'authentication'),
+        (r'\b(error|exception|bug|fix)\b', 'debugging'),
+        (r'\b(refactor|optimize|performance)\b', 'optimization'),
+        (r'\b(deploy|ci|cd|pipeline)\b', 'deployment')
+    ]
+
+    text_lower = text.lower()
+    seen_concepts = set()
+
+    for pattern, concept in concept_patterns:
+        if re.search(pattern, text_lower) and concept not in seen_concepts:
+            concepts.append(concept)
+            seen_concepts.add(concept)
+            if len(concepts) >= MAX_CONCEPTS:
+                break
+
+    return concepts