claude-self-reflect 5.0.4 → 5.0.6

package/scripts/message_processors.py

@@ -0,0 +1,248 @@
+"""
+Message processor classes for handling different message types in JSONL import.
+Refactored from extract_metadata_single_pass to reduce complexity.
+"""
+
+import re
+import ast
+import logging
+from abc import ABC, abstractmethod
+from typing import Dict, Any, List, Set, Optional
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+# Constants for metadata limits (can be overridden via environment variables)
+import os
+
+MAX_CONCEPTS = int(os.getenv("MAX_CONCEPTS", "10"))
+MAX_AST_ELEMENTS = int(os.getenv("MAX_AST_ELEMENTS", "30"))
+MAX_CODE_BLOCKS = int(os.getenv("MAX_CODE_BLOCKS", "5"))
+MAX_ELEMENTS_PER_BLOCK = int(os.getenv("MAX_ELEMENTS_PER_BLOCK", "10"))
+MAX_FILES_ANALYZED = int(os.getenv("MAX_FILES_ANALYZED", "20"))
+MAX_FILES_EDITED = int(os.getenv("MAX_FILES_EDITED", "20"))
+MAX_TOOLS_USED = int(os.getenv("MAX_TOOLS_USED", "15"))
+MAX_CONCEPT_MESSAGES = int(os.getenv("MAX_CONCEPT_MESSAGES", "50"))
+
+
+class MessageProcessor(ABC):
+    """Abstract base class for message processing."""
+
+    @abstractmethod
+    def process(self, item: Any, metadata: Dict[str, Any]) -> Optional[str]:
+        """Process a message item and update metadata."""
+        pass
+
+
+class TextMessageProcessor(MessageProcessor):
+    """Process text messages and extract code blocks."""
+
+    def process(self, item: Dict[str, Any], metadata: Dict[str, Any]) -> Optional[str]:
+        """Process text content and extract code blocks with AST elements."""
+        if item.get('type') != 'text':
+            return None
+
+        text_content = item.get('text', '')
+
+        # Check for code blocks
+        if '```' in text_content:
+            metadata['has_code_blocks'] = True
+            self._extract_code_ast_elements(text_content, metadata)
+
+        return text_content
+
+    def _extract_code_ast_elements(self, text: str, metadata: Dict[str, Any]):
+        """Extract AST elements from code blocks in text."""
+        if 'ast_elements' not in metadata:
+            metadata['ast_elements'] = []
+
+        if len(metadata['ast_elements']) >= MAX_AST_ELEMENTS:
+            return
+
+        # More permissive regex to handle various fence formats
+        code_blocks = re.findall(r'```[^`\n]*\n?(.*?)```', text, re.DOTALL)
+
+        for code_block in code_blocks[:MAX_CODE_BLOCKS]:
+            if len(metadata['ast_elements']) >= MAX_AST_ELEMENTS:
+                break
+
+            ast_elems = extract_ast_elements(code_block)
+            for elem in list(ast_elems)[:MAX_ELEMENTS_PER_BLOCK]:
+                if elem not in metadata['ast_elements'] and len(metadata['ast_elements']) < MAX_AST_ELEMENTS:
+                    metadata['ast_elements'].append(elem)
+
+
+class ThinkingMessageProcessor(MessageProcessor):
+    """Process thinking messages."""
+
+    def process(self, item: Dict[str, Any], metadata: Dict[str, Any]) -> Optional[str]:
+        """Process thinking content."""
+        if item.get('type') != 'thinking':
+            return None
+
+        return item.get('thinking', '')
+
+
+class ToolMessageProcessor(MessageProcessor):
+    """Process tool use messages and extract file references."""
+
+    def process(self, item: Dict[str, Any], metadata: Dict[str, Any]) -> Optional[str]:
+        """Process tool use and extract file references."""
+        if item.get('type') != 'tool_use':
+            return None
+
+        tool_name = item.get('name', '')
+
+        # Track tool usage
+        if 'tools_used' not in metadata:
+            metadata['tools_used'] = []
+
+        if tool_name and tool_name not in metadata['tools_used']:
+            if len(metadata['tools_used']) < MAX_TOOLS_USED:
+                metadata['tools_used'].append(tool_name)
+
+        # Extract file references
+        if 'input' in item:
+            self._extract_file_references(item['input'], tool_name, metadata)
+
+        # Return tool use as text
+        tool_input = str(item.get('input', ''))[:500]
+        return f"[Tool: {tool_name}] {tool_input}"
+
+    def _extract_file_references(self, input_data: Any, tool_name: str, metadata: Dict[str, Any]):
+        """Extract file references from tool input."""
+        if not isinstance(input_data, dict):
+            return
+
+        # Initialize metadata lists if not present
+        if 'files_edited' not in metadata:
+            metadata['files_edited'] = []
+        if 'files_analyzed' not in metadata:
+            metadata['files_analyzed'] = []
+
+        is_edit = tool_name in ['Edit', 'Write', 'MultiEdit', 'NotebookEdit']
+
+        # Check file_path field
+        if 'file_path' in input_data:
+            file_ref = input_data['file_path']
+            if is_edit:
+                if file_ref not in metadata['files_edited'] and len(metadata['files_edited']) < MAX_FILES_EDITED:
+                    metadata['files_edited'].append(file_ref)
+            else:
+                if file_ref not in metadata['files_analyzed'] and len(metadata['files_analyzed']) < MAX_FILES_ANALYZED:
+                    metadata['files_analyzed'].append(file_ref)
+
+        # Check path field (for non-edit tools)
+        if 'path' in input_data and not is_edit:
+            file_ref = input_data['path']
+            if file_ref not in metadata['files_analyzed'] and len(metadata['files_analyzed']) < MAX_FILES_ANALYZED:
+                metadata['files_analyzed'].append(file_ref)
+
+
+class ToolResultProcessor(MessageProcessor):
+    """Process tool result messages."""
+
+    def process(self, item: Any, metadata: Dict[str, Any]) -> Optional[str]:
+        """Process tool results."""
+        # Handle both dict items and top-level tool results
+        if isinstance(item, dict):
+            if item.get('type') == 'tool_result':
+                result_content = str(item.get('content', ''))[:1000]
+                return f"[Result] {result_content}"
+            elif item.get('type') == 'tool_use':
+                # Already handled by ToolMessageProcessor
+                return None
+
+        return None
+
+
+class MessageProcessorFactory:
+    """Factory for creating appropriate message processors."""
+
+    def __init__(self):
+        self.processors = {
+            'text': TextMessageProcessor(),
+            'thinking': ThinkingMessageProcessor(),
+            'tool_use': ToolMessageProcessor(),
+            'tool_result': ToolResultProcessor()
+        }
+
+    def get_processor(self, message_type: str) -> Optional[MessageProcessor]:
+        """Get the appropriate processor for a message type."""
+        return self.processors.get(message_type)
+
+    def process_content(self, content: Any, metadata: Dict[str, Any]) -> str:
+        """Process content of various types and return text representation."""
+        text_parts = []
+
+        if isinstance(content, list):
+            for item in content:
+                if isinstance(item, dict):
+                    item_type = item.get('type', '')
+                    processor = self.get_processor(item_type)
+                    if processor:
+                        text = processor.process(item, metadata)
+                        if text:
+                            text_parts.append(text)
+                elif isinstance(item, str):
+                    text_parts.append(item)
+        elif isinstance(content, str):
+            text_parts.append(content)
+
+        return '\n'.join(text_parts)
+
+
+def extract_ast_elements(code_text: str) -> Set[str]:
+    """Extract AST elements from Python code."""
+    elements = set()
+
+    try:
+        tree = ast.parse(code_text)
+        for node in ast.walk(tree):
+            if isinstance(node, ast.FunctionDef):
+                elements.add(f"func:{node.name}")
+            elif isinstance(node, ast.ClassDef):
+                elements.add(f"class:{node.name}")
+            elif isinstance(node, ast.Import):
+                for alias in node.names:
+                    elements.add(f"import:{alias.name}")
+            elif isinstance(node, ast.ImportFrom):
+                module = node.module or ''
+                for alias in node.names:
+                    elements.add(f"from:{module}.{alias.name}")
+    except (SyntaxError, ValueError):
+        # Not Python code or invalid syntax
+        pass
+
+    return elements
+
+
+def extract_concepts(text: str) -> List[str]:
+    """Extract key concepts from text using simple heuristics."""
+    concepts = []
+
+    # Common programming concepts
+    concept_patterns = [
+        (r'\b(async|await|promise|future)\b', 'async-programming'),
+        (r'\b(test|spec|jest|pytest|unittest)\b', 'testing'),
+        (r'\b(docker|container|kubernetes|k8s)\b', 'containerization'),
+        (r'\b(api|rest|graphql|endpoint)\b', 'api-development'),
+        (r'\b(react|vue|angular|svelte)\b', 'frontend-framework'),
+        (r'\b(database|sql|postgres|mysql|mongodb)\b', 'database'),
+        (r'\b(auth|authentication|oauth|jwt)\b', 'authentication'),
+        (r'\b(error|exception|bug|fix)\b', 'debugging'),
+        (r'\b(refactor|optimize|performance)\b', 'optimization'),
+        (r'\b(deploy|ci|cd|pipeline)\b', 'deployment')
+    ]
+
+    text_lower = text.lower()
+    seen_concepts = set()
+
+    for pattern, concept in concept_patterns:
+        if re.search(pattern, text_lower) and concept not in seen_concepts:
+            concepts.append(concept)
+            seen_concepts.add(concept)
+            if len(concepts) >= MAX_CONCEPTS:
+                break
+
+    return concepts

package/scripts/metadata_extractor.py

@@ -0,0 +1,262 @@
+"""
+Metadata extractor using message processors to reduce complexity.
+Refactored from extract_metadata_single_pass function.
+"""
+
+import json
+import os
+import logging
+from pathlib import Path
+from typing import Dict, Any, Tuple, Optional
+from datetime import datetime
+
+from message_processors import (
+    MessageProcessorFactory,
+    extract_concepts,
+    MAX_CONCEPT_MESSAGES,
+    MAX_FILES_ANALYZED,
+    MAX_FILES_EDITED,
+    MAX_TOOLS_USED,
+    MAX_AST_ELEMENTS
+)
+
+logger = logging.getLogger(__name__)
+
+
+class MetadataExtractor:
+    """Extract metadata from JSONL conversation files."""
+
+    def __init__(self):
+        self.processor_factory = MessageProcessorFactory()
+
+    def extract_metadata_from_file(self, file_path: str) -> Tuple[Dict[str, Any], str, int]:
+        """
+        Extract metadata from a JSONL file in a single pass.
+        Returns: (metadata, first_timestamp, message_count)
+        """
+        metadata = self._initialize_metadata()
+        first_timestamp = None
+        message_count = 0
+        all_text = []
+
+        try:
+            with open(file_path, 'r', encoding='utf-8') as f:
+                for line in f:
+                    if not line.strip():
+                        continue
+
+                    result = self._process_line(line, metadata)
+                    if result:
+                        text_content, is_message = result
+
+                        # Update timestamp and counts
+                        if first_timestamp is None:
+                            first_timestamp = self._extract_timestamp(line)
+
+                        if is_message:
+                            message_count += 1
+
+                        if text_content:
+                            # Limit text accumulation to prevent memory issues
+                            if len(all_text) < MAX_CONCEPT_MESSAGES:
+                                all_text.append(text_content[:1000])
+
+        except (IOError, OSError) as e:
+            logger.warning(f"Error reading file {file_path}: {e}")
+        except (json.JSONDecodeError, ValueError) as e:
+            logger.warning(f"Error parsing JSON in {file_path}: {e}")
+        except Exception as e:
+            logger.error(f"Unexpected error extracting metadata from {file_path}: {e}")
+
+        # Post-process collected data
+        self._post_process_metadata(metadata, all_text, file_path)
+
+        # Apply limits to arrays
+        self._apply_metadata_limits(metadata)
+
+        return metadata, first_timestamp or datetime.now().isoformat(), message_count
+
+    def _initialize_metadata(self) -> Dict[str, Any]:
+        """Initialize empty metadata structure."""
+        return {
+            "files_analyzed": [],
+            "files_edited": [],
+            "tools_used": [],
+            "concepts": [],
+            "ast_elements": [],
+            "has_code_blocks": False,
+            "total_messages": 0,
+            "project_path": None,
+            "pattern_analysis": {},
+            "avg_quality_score": 0.0
+        }
+
+    def _process_line(self, line: str, metadata: Dict[str, Any]) -> Optional[Tuple[str, bool]]:
+        """
+        Process a single line from the JSONL file.
+        Returns: (text_content, is_message) or None
+        """
+        try:
+            data = json.loads(line)
+
+            # Extract project path from cwd
+            if metadata["project_path"] is None and 'cwd' in data:
+                metadata["project_path"] = data.get('cwd')
+
+            # Handle message entries
+            if 'message' in data and data['message']:
+                return self._process_message_entry(data['message'], metadata)
+
+            # Handle top-level tool entries
+            entry_type = data.get('type')
+            if entry_type in ('tool_result', 'tool_use'):
+                return self._process_tool_entry(data, metadata)
+
+        except json.JSONDecodeError:
+            # Expected for non-JSON lines, skip silently
+            pass
+        except (KeyError, TypeError, ValueError) as e:
+            # Log specific parsing errors for debugging
+            logger.debug(f"Error parsing line: {e}")
+
+        return None
+
+    def _process_message_entry(self, message: Dict[str, Any], metadata: Dict[str, Any]) -> Optional[Tuple[str, bool]]:
+        """Process a message entry."""
+        role = message.get('role')
+        content = message.get('content')
+
+        if not role or not content:
+            return None
+
+        # Check if it's a countable message
+        is_user_or_assistant = role in ['user', 'assistant']
+
+        # Process content
+        text_content = self.processor_factory.process_content(content, metadata)
+
+        return text_content, is_user_or_assistant
+
+    def _process_tool_entry(self, data: Dict[str, Any], metadata: Dict[str, Any]) -> Optional[Tuple[str, bool]]:
+        """Process a top-level tool entry."""
+        entry_type = data.get('type')
+        text_parts = []
+
+        if entry_type == 'tool_use':
+            tool_name = data.get('name', 'unknown')
+            tool_input = str(data.get('input', ''))[:500]
+            text_parts.append(f"[Tool: {tool_name}] {tool_input}")
+
+            # Track tool usage
+            if tool_name and tool_name not in metadata['tools_used']:
+                metadata['tools_used'].append(tool_name)
+
+        elif entry_type == 'tool_result':
+            result_content = self._extract_tool_result_content(data)
+            text_parts.append(f"[Result] {result_content[:1000]}")
+
+        content = "\n".join(text_parts)
+        # Tool entries should not count as messages (only user/assistant messages count)
+        return (content, False) if content else None
+
+    def _extract_tool_result_content(self, data: Dict[str, Any]) -> str:
+        """Extract content from tool result data."""
+        result_content = data.get('content')
+
+        if isinstance(result_content, list):
+            flat = []
+            for item in result_content:
+                if isinstance(item, dict) and item.get('type') == 'text':
+                    flat.append(item.get('text', ''))
+                elif isinstance(item, str):
+                    flat.append(item)
+            result_content = "\n".join(flat)
+
+        if not result_content:
+            result_content = data.get('result', '')
+
+        return str(result_content)
+
+    def _extract_timestamp(self, line: str) -> Optional[str]:
+        """Extract timestamp from a line if present."""
+        try:
+            data = json.loads(line)
+            return data.get('timestamp')
+        except (json.JSONDecodeError, TypeError) as e:
+            logger.debug(f"Failed to extract timestamp: {e}")
+            return None
+
+    def _post_process_metadata(self, metadata: Dict[str, Any], all_text: list, file_path: str):
+        """Post-process collected metadata."""
+        # Extract concepts from collected text
+        if all_text:
+            combined_text = ' '.join(all_text[:MAX_CONCEPT_MESSAGES])
+            metadata['concepts'] = extract_concepts(combined_text)
+
+        # Run AST-GREP pattern analysis if available
+        self._run_pattern_analysis(metadata)
+
+    def _run_pattern_analysis(self, metadata: Dict[str, Any]):
+        """Run AST-GREP pattern analysis on mentioned files."""
+        pattern_quality = {}
+        avg_quality_score = 0.0
+
+        try:
+            # Update patterns first
+            from update_patterns import check_and_update_patterns
+            check_and_update_patterns()
+
+            # Import analyzer
+            from ast_grep_final_analyzer import FinalASTGrepAnalyzer
+            analyzer = FinalASTGrepAnalyzer()
+
+            # Analyze files
+            files_to_analyze = list(set(
+                metadata['files_edited'] + metadata['files_analyzed'][:10]
+            ))
+            quality_scores = []
+
+            for file_path in files_to_analyze:
+                # Expand file path for proper checking
+                expanded_path = os.path.expanduser(file_path) if file_path.startswith('~') else file_path
+                if self._is_code_file(expanded_path) and os.path.exists(expanded_path):
+                    try:
+                        result = analyzer.analyze_file(expanded_path)
+                        metrics = result['quality_metrics']
+                        pattern_quality[file_path] = {
+                            'score': metrics['quality_score'],
+                            'good_patterns': metrics['good_patterns_found'],
+                            'bad_patterns': metrics['bad_patterns_found'],
+                            'issues': metrics['total_issues']
+                        }
+                        quality_scores.append(metrics['quality_score'])
+                    except (IOError, OSError) as e:
+                        logger.debug(f"Could not read file {file_path}: {e}")
+                    except (KeyError, ValueError) as e:
+                        logger.debug(f"Error parsing AST results for {file_path}: {e}")
+                    except Exception as e:
+                        logger.warning(f"Unexpected error analyzing {file_path}: {e}")
+
+            # Calculate average quality
+            if quality_scores:
+                avg_quality_score = sum(quality_scores) / len(quality_scores)
+
+        except Exception as e:
+            logger.debug(f"AST analysis not available: {e}")
+
+        metadata['pattern_analysis'] = pattern_quality
+        metadata['avg_quality_score'] = round(avg_quality_score, 3)
+
+    def _is_code_file(self, file_path: str) -> bool:
+        """Check if file is a code file."""
+        if not file_path:
+            return False
+        extensions = ['.py', '.ts', '.js', '.tsx', '.jsx']
+        return any(file_path.endswith(ext) for ext in extensions)
+
+    def _apply_metadata_limits(self, metadata: Dict[str, Any]):
+        """Apply size limits to metadata arrays."""
+        metadata['files_analyzed'] = metadata['files_analyzed'][:MAX_FILES_ANALYZED]
+        metadata['files_edited'] = metadata['files_edited'][:MAX_FILES_EDITED]
+        metadata['tools_used'] = metadata['tools_used'][:MAX_TOOLS_USED]
+        metadata['ast_elements'] = metadata['ast_elements'][:MAX_AST_ELEMENTS]

package/scripts/watcher-loop.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+# Watcher loop for Docker container
+# Runs the streaming-watcher.py with HOT/WARM/COLD prioritization
+
+# Don't use set -e in retry loops - it can cause premature exits
+
+echo "Starting Claude Self-Reflect Streaming Watcher v3.0.0"
+echo "HOT/WARM/COLD prioritization enabled"
+echo "=========================================="
+
+# Ensure config directory exists
+mkdir -p /config
+
+# Set Python path to include scripts directory
+export PYTHONPATH=/app/scripts:$PYTHONPATH
+
+# Main loop - restart on failure with backoff
+RETRY_COUNT=0
+MAX_RETRIES=10
+BACKOFF_SECONDS=5
+
+while true; do
+    echo "[$(date)] Starting watcher (attempt $((RETRY_COUNT + 1))/$MAX_RETRIES)..."
+    
+    # Run the streaming watcher
+    python /app/scripts/streaming-watcher.py
+    EXIT_CODE=$?
+    
+    if [ $EXIT_CODE -eq 0 ]; then
+        echo "[$(date)] Watcher exited cleanly"
+        RETRY_COUNT=0
+        BACKOFF_SECONDS=5
+    else
+        echo "[$(date)] Watcher exited with code $EXIT_CODE"
+        
+        RETRY_COUNT=$((RETRY_COUNT + 1))
+        if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then
+            echo "[$(date)] Maximum retries reached. Exiting."
+            exit 1
+        fi
+        
+        # Add jitter to prevent thundering herd (±20% of backoff)
+        JITTER=$(( (RANDOM % (BACKOFF_SECONDS / 5 + 1)) - (BACKOFF_SECONDS / 10) ))
+        SLEEP_TIME=$((BACKOFF_SECONDS + JITTER))
+        [ $SLEEP_TIME -lt 1 ] && SLEEP_TIME=1
+        
+        echo "[$(date)] Restarting in $SLEEP_TIME seconds (base: $BACKOFF_SECONDS, jitter: $JITTER)..."
+        sleep $SLEEP_TIME
+        
+        # Exponential backoff (max 300 seconds)
+        BACKOFF_SECONDS=$((BACKOFF_SECONDS * 2))
+        if [ $BACKOFF_SECONDS -gt 300 ]; then
+            BACKOFF_SECONDS=300
+        fi
+    fi
+done