claude-self-reflect 3.3.0 → 4.0.0

package/scripts/streaming-watcher.py

@@ -484,7 +484,20 @@ class QdrantService:
     
     def __init__(self, config: Config, embedding_provider: EmbeddingProvider):
         self.config = config
-        self.client = AsyncQdrantClient(url=config.qdrant_url)
+
+        # Security: Validate Qdrant URL for remote connections
+        from urllib.parse import urlparse
+        parsed = urlparse(config.qdrant_url)
+        host = (parsed.hostname or "").lower()
+
+        if config.require_tls_for_remote and host not in ("localhost", "127.0.0.1", "qdrant") and parsed.scheme != "https":
+            raise ValueError(f"Insecure QDRANT_URL for remote host: {config.qdrant_url} (use https:// or set QDRANT_REQUIRE_TLS_FOR_REMOTE=false)")
+
+        # Initialize with API key if provided
+        self.client = AsyncQdrantClient(
+            url=config.qdrant_url,
+            api_key=config.qdrant_api_key if hasattr(config, 'qdrant_api_key') else None
+        )
         self.embedding_provider = embedding_provider
         self._collection_cache: Dict[str, float] = {}
         self.request_semaphore = asyncio.Semaphore(config.max_concurrent_qdrant)
@@ -979,7 +992,66 @@ class StreamingWatcher:
                         text_parts.append(item.get('text', ''))
             return ' '.join(text_parts)
         return str(content) if content else ''
-    
+
+    def _update_quality_cache(self, pattern_analysis: Dict, avg_score: float, project_name: str = None):
+        """Update quality cache file for statusline display - PER PROJECT."""
+        try:
+            # Determine project name from current context or use provided
+            if not project_name:
+                # Try to infer from analyzed files
+                files = pattern_analysis.get('files', [])
+                if files and len(files) > 0:
+                    # Extract project from first file path
+                    first_file = Path(files[0])
+                    # Walk up to find .git directory or use immediate parent
+                    for parent in first_file.parents:
+                        if (parent / '.git').exists():
+                            project_name = parent.name
+                            break
+                    else:
+                        project_name = 'unknown'
+                else:
+                    project_name = 'unknown'
+
+            # Sanitize project name for filename
+            safe_project_name = project_name.replace('/', '-').replace(' ', '_')
+            cache_dir = Path.home() / ".claude-self-reflect" / "quality_cache"
+            cache_dir.mkdir(exist_ok=True, parents=True)
+            cache_file = cache_dir / f"{safe_project_name}.json"
+
+            # Calculate total issues from pattern analysis
+            total_issues = sum(p.get('count', 0) for p in pattern_analysis.get('issues', []))
+
+            # Determine grade based on score
+            if avg_score >= 0.95:
+                grade = 'A+' if total_issues < 10 else 'A'
+            elif avg_score >= 0.8:
+                grade = 'B'
+            elif avg_score >= 0.6:
+                grade = 'C'
+            else:
+                grade = 'D'
+
+            cache_data = {
+                'status': 'success',
+                'session_id': 'watcher',
+                'timestamp': datetime.now().isoformat(),
+                'summary': {
+                    'files_analyzed': len(pattern_analysis.get('files', [])),
+                    'avg_quality_score': round(avg_score, 3),
+                    'total_issues': total_issues,
+                    'quality_grade': grade
+                }
+            }
+
+            with open(cache_file, 'w') as f:
+                json.dump(cache_data, f, indent=2)
+
+            logger.debug(f"Updated quality cache: {grade}/{total_issues}")
+
+        except Exception as e:
+            logger.debug(f"Failed to update quality cache: {e}")
+
     async def process_file(self, file_path: Path) -> bool:
         """Process a single file."""
         try:
@@ -1034,7 +1106,70 @@ class StreamingWatcher:
             
             # Extract metadata
             tool_usage = extract_tool_usage_from_conversation(all_messages)
-            
+
+            # MANDATORY AST-GREP Analysis for HOT files
+            pattern_analysis = {}
+            avg_quality_score = 0.0
+            freshness_level, _ = self.categorize_freshness(file_path)
+
+            # Analyze code quality for HOT files (current session)
+            if freshness_level == FreshnessLevel.HOT and (tool_usage.get('files_edited') or tool_usage.get('files_analyzed')):
+                try:
+                    # Import analyzer (lazy import to avoid startup overhead)
+                    from ast_grep_final_analyzer import FinalASTGrepAnalyzer
+                    from update_patterns import check_and_update_patterns
+
+                    # Update patterns (24h cache, <100ms)
+                    check_and_update_patterns()
+
+                    # Create analyzer
+                    if not hasattr(self, '_ast_analyzer'):
+                        self._ast_analyzer = FinalASTGrepAnalyzer()
+
+                    # Analyze edited files from this session
+                    files_to_analyze = list(set(
+                        tool_usage.get('files_edited', [])[:5] +
+                        tool_usage.get('files_analyzed', [])[:5]
+                    ))
+
+                    quality_scores = []
+                    for file_ref in files_to_analyze:
+                        if file_ref and any(file_ref.endswith(ext) for ext in ['.py', '.ts', '.js', '.tsx', '.jsx']):
+                            try:
+                                if os.path.exists(file_ref):
+                                    result = self._ast_analyzer.analyze_file(file_ref)
+                                    metrics = result['quality_metrics']
+                                    pattern_analysis[file_ref] = {
+                                        'score': metrics['quality_score'],
+                                        'good_patterns': metrics['good_patterns_found'],
+                                        'bad_patterns': metrics['bad_patterns_found'],
+                                        'issues': metrics['total_issues']
+                                    }
+                                    quality_scores.append(metrics['quality_score'])
+
+                                    # Log quality issues for HOT files
+                                    if metrics['quality_score'] < 0.6:
+                                        logger.warning(f"⚠️ Quality issue in {os.path.basename(file_ref)}: {metrics['quality_score']:.1%} ({metrics['total_issues']} issues)")
+                            except Exception as e:
+                                logger.debug(f"Could not analyze {file_ref}: {e}")
+
+                    if quality_scores:
+                        avg_quality_score = sum(quality_scores) / len(quality_scores)
+                        logger.info(f"📊 Session quality: {avg_quality_score:.1%} for {len(quality_scores)} files")
+
+                        # Update quality cache for statusline - watcher handles this automatically!
+                        # Pass project name from current file being processed
+                        project_name = file_path.parent.name if file_path else None
+                        self._update_quality_cache(pattern_analysis, avg_quality_score, project_name)
+
+                except Exception as e:
+                    logger.debug(f"AST analysis not available: {e}")
+
+            # Add pattern analysis to tool_usage metadata
+            if pattern_analysis:
+                tool_usage['pattern_analysis'] = pattern_analysis
+                tool_usage['avg_quality_score'] = round(avg_quality_score, 3)
+
             # Build text
             text_parts = []
             for msg in all_messages:
@@ -1043,7 +1178,7 @@ class StreamingWatcher:
                 text = self._extract_message_text(content)
                 if text:
                     text_parts.append(f"{role}: {text}")
-            
+
             combined_text = "\n\n".join(text_parts)
             if not combined_text.strip():
                 logger.warning(f"No textual content in {file_path}, marking as processed")
@@ -1057,7 +1192,7 @@ class StreamingWatcher:
                 }
                 self.stats["files_processed"] += 1
                 return True
-            
+
             concepts = extract_concepts(combined_text, tool_usage)
             
             # Now we know we have content, ensure collection exists

package/scripts/update_patterns.py

@@ -0,0 +1,334 @@
+#!/usr/bin/env python3
+"""
+Auto-update AST-GREP patterns from official catalog.
+MANDATORY feature - runs on every import to ensure latest patterns.
+Fast execution: <1 second with caching.
+"""
+
+import os
+import json
+import yaml
+import re
+import hashlib
+from pathlib import Path
+from datetime import datetime, timedelta
+from typing import Dict, List, Any, Optional
+import subprocess
+import tempfile
+import shutil
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Configuration
+CACHE_DIR = Path.home() / ".claude-self-reflect" / "cache" / "patterns"
+CACHE_FILE = CACHE_DIR / "pattern_cache.json"
+REGISTRY_FILE = Path(__file__).parent / "unified_registry.json"
+CATALOG_REPO = "https://github.com/ast-grep/ast-grep.github.io.git"
+CATALOG_PATH = "website/catalog"
+CACHE_HOURS = 24  # Check for updates once per day
+
+# Ensure cache directory exists
+CACHE_DIR.mkdir(parents=True, exist_ok=True)
+
+
+class PatternUpdater:
+    """Updates AST-GREP patterns from official catalog."""
+
+    def __init__(self):
+        self.patterns = {}
+        self.stats = {
+            'total_patterns': 0,
+            'new_patterns': 0,
+            'updated_patterns': 0,
+            'languages': set()
+        }
+
+    def should_update(self) -> bool:
+        """Check if patterns need updating based on cache age."""
+        if not CACHE_FILE.exists():
+            return True
+
+        try:
+            with open(CACHE_FILE, 'r') as f:
+                cache = json.load(f)
+
+            cached_time = datetime.fromisoformat(cache.get('timestamp', '2000-01-01'))
+            if datetime.now() - cached_time > timedelta(hours=CACHE_HOURS):
+                return True
+
+            # Also check if registry file is missing
+            if not REGISTRY_FILE.exists():
+                return True
+
+            return False
+        except:
+            return True
+
+    def fetch_catalog_patterns(self) -> Dict[str, List[Dict]]:
+        """Fetch latest patterns from AST-GREP GitHub catalog."""
+        patterns_by_lang = {}
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            repo_path = Path(tmpdir) / "ast-grep-catalog"
+
+            try:
+                # Clone or pull the repository (shallow clone for speed)
+                logger.info("Fetching latest AST-GREP patterns from GitHub...")
+                # Use shorter timeout to avoid blocking analysis
+                timeout = int(os.environ.get("AST_GREP_CATALOG_TIMEOUT", "10"))
+                subprocess.run(
+                    ["git", "clone", "--depth", "1", "--single-branch",
+                     CATALOG_REPO, str(repo_path)],
+                    check=True,
+                    capture_output=True,
+                    timeout=timeout
+                )
+
+                catalog_dir = repo_path / CATALOG_PATH
+
+                # Process each language directory
+                for lang_dir in catalog_dir.iterdir():
+                    if lang_dir.is_dir() and not lang_dir.name.startswith('.'):
+                        language = lang_dir.name
+                        patterns_by_lang[language] = []
+                        self.stats['languages'].add(language)
+
+                        # Process each pattern file
+                        for pattern_file in lang_dir.glob("*.md"):
+                            if pattern_file.name == "index.md":
+                                continue
+
+                            pattern = self._parse_pattern_file(pattern_file, language)
+                            if pattern:
+                                patterns_by_lang[language].append(pattern)
+                                self.stats['total_patterns'] += 1
+
+                logger.info(f"Fetched {self.stats['total_patterns']} patterns for {len(self.stats['languages'])} languages")
+
+            except subprocess.TimeoutExpired:
+                logger.warning("GitHub fetch timed out, using cached patterns")
+                return {}
+            except Exception as e:
+                logger.warning(f"Failed to fetch from GitHub: {e}, using cached patterns")
+                return {}
+
+        return patterns_by_lang
+
+    def _parse_pattern_file(self, file_path: Path, language: str) -> Optional[Dict]:
+        """Parse a single pattern file from the catalog."""
+        try:
+            content = file_path.read_text()
+
+            # Extract YAML block
+            yaml_match = re.search(r'```yaml\n(.*?)\n```', content, re.DOTALL)
+            if not yaml_match:
+                return None
+
+            yaml_content = yaml_match.group(1)
+            pattern_data = yaml.safe_load(yaml_content)
+
+            # Extract metadata
+            title_match = re.search(r'^## (.+?)(?:\s*<Badge.*?>)?$', content, re.MULTILINE)
+            title = title_match.group(1).strip() if title_match else file_path.stem
+
+            # Extract description
+            desc_match = re.search(r'### Description\n\n(.+?)(?=\n###|\n```|\Z)', content, re.DOTALL)
+            description = desc_match.group(1).strip() if desc_match else ""
+
+            # Build pattern object
+            pattern = {
+                'id': pattern_data.get('id', file_path.stem),
+                'title': title,
+                'description': description,
+                'language': pattern_data.get('language', language),
+                'file': file_path.name,
+                'has_fix': 'fix' in pattern_data
+            }
+
+            # Extract rule
+            if 'rule' in pattern_data:
+                rule = pattern_data['rule']
+                if isinstance(rule, dict):
+                    if 'pattern' in rule:
+                        pattern['pattern'] = rule['pattern']
+                    if 'any' in rule:
+                        pattern['patterns'] = rule['any']
+                        pattern['match_type'] = 'any'
+                    if 'all' in rule:
+                        pattern['patterns'] = rule['all']
+                        pattern['match_type'] = 'all'
+                    if 'inside' in rule:
+                        pattern['inside'] = rule['inside']
+
+            # Add fix if present
+            if 'fix' in pattern_data:
+                pattern['fix'] = pattern_data['fix']
+
+            # Determine quality based on type
+            pattern['quality'] = self._determine_quality(pattern)
+            pattern['weight'] = self._calculate_weight(pattern)
+
+            return pattern
+
+        except Exception as e:
+            logger.debug(f"Failed to parse {file_path}: {e}")
+            return None
+
+    def _determine_quality(self, pattern: Dict) -> str:
+        """Determine pattern quality."""
+        if pattern.get('has_fix'):
+            return 'good'
+
+        # Patterns that detect issues are "bad" (they find bad code)
+        if any(word in pattern.get('id', '').lower()
+               for word in ['no-', 'missing-', 'avoid-', 'deprecated']):
+            return 'bad'
+
+        return 'neutral'
+
+    def _calculate_weight(self, pattern: Dict) -> int:
+        """Calculate pattern weight for scoring."""
+        quality = pattern.get('quality', 'neutral')
+        weights = {
+            'good': 3,
+            'neutral': 1,
+            'bad': -3
+        }
+        return weights.get(quality, 1)
+
+    def merge_with_custom_patterns(self, catalog_patterns: Dict) -> Dict:
+        """Merge catalog patterns with custom local patterns."""
+        # Load existing registry if it exists
+        existing_patterns = {}
+        if REGISTRY_FILE.exists():
+            try:
+                with open(REGISTRY_FILE, 'r') as f:
+                    registry = json.load(f)
+                    existing_patterns = registry.get('patterns', {})
+            except:
+                pass
+
+        # Keep custom Python patterns (our manual additions)
+        custom_categories = [
+            'python_async', 'python_error_handling', 'python_logging',
+            'python_typing', 'python_antipatterns', 'python_qdrant', 'python_mcp'
+        ]
+
+        merged = {}
+        for category in custom_categories:
+            if category in existing_patterns:
+                merged[category] = existing_patterns[category]
+
+        # Add catalog patterns
+        for language, patterns in catalog_patterns.items():
+            category_name = f"{language}_catalog"
+            merged[category_name] = patterns
+
+        return merged
+
+    def save_registry(self, patterns: Dict):
+        """Save updated pattern registry."""
+        registry = {
+            'source': 'unified-ast-grep-auto-updated',
+            'version': '3.0.0',
+            'timestamp': datetime.now().isoformat(),
+            'patterns': patterns,
+            'stats': {
+                'total_patterns': sum(len(p) for p in patterns.values()),
+                'categories': list(patterns.keys()),
+                'languages': list(self.stats['languages']),
+                'last_update': datetime.now().isoformat()
+            }
+        }
+
+        with open(REGISTRY_FILE, 'w') as f:
+            json.dump(registry, f, indent=2)
+
+        logger.info(f"Saved {registry['stats']['total_patterns']} patterns to {REGISTRY_FILE}")
+
+    def update_cache(self):
+        """Update cache file with timestamp."""
+        cache_data = {
+            'timestamp': datetime.now().isoformat(),
+            'stats': {
+                'total_patterns': self.stats['total_patterns'],
+                'languages': list(self.stats['languages'])
+            }
+        }
+
+        with open(CACHE_FILE, 'w') as f:
+            json.dump(cache_data, f)
+
+    def update_patterns(self, force: bool = False) -> bool:
+        """Main update function - FAST with caching."""
+        # Check if update needed (< 10ms)
+        if not force and not self.should_update():
+            logger.debug("Patterns are up to date (cached)")
+            return False
+
+        logger.info("Updating AST-GREP patterns...")
+
+        # Fetch from GitHub (only when cache expired)
+        catalog_patterns = self.fetch_catalog_patterns()
+
+        if catalog_patterns:
+            # Merge with custom patterns
+            merged_patterns = self.merge_with_custom_patterns(catalog_patterns)
+
+            # Save updated registry
+            self.save_registry(merged_patterns)
+
+            # Update cache timestamp
+            self.update_cache()
+
+            logger.info(f"✅ Pattern update complete: {self.stats['total_patterns']} patterns")
+            return True
+        else:
+            logger.info("Using existing patterns (GitHub unavailable)")
+            return False
+
+
+def check_and_update_patterns(force: bool = False) -> bool:
+    """
+    Quick pattern update check - MANDATORY but FAST.
+    Called on every import, uses 24-hour cache.
+    """
+    updater = PatternUpdater()
+    return updater.update_patterns(force=force)
+
+
+def install_time_update():
+    """Run during package installation - forces update."""
+    logger.info("Installing AST-GREP patterns...")
+    updater = PatternUpdater()
+    updater.update_patterns(force=True)
+
+
+if __name__ == "__main__":
+    import sys
+
+    # Allow --force flag for manual updates
+    force = "--force" in sys.argv
+
+    if force:
+        print("Forcing pattern update from GitHub...")
+    else:
+        print("Checking for pattern updates (24-hour cache)...")
+
+    success = check_and_update_patterns(force=force)
+
+    if success:
+        print("✅ Patterns updated successfully")
+    else:
+        print("✅ Patterns are up to date")
+
+    # Show stats
+    if REGISTRY_FILE.exists():
+        with open(REGISTRY_FILE, 'r') as f:
+            registry = json.load(f)
+            stats = registry.get('stats', {})
+            print(f"   Total patterns: {stats.get('total_patterns', 0)}")
+            print(f"   Languages: {', '.join(stats.get('languages', []))}")
+            print(f"   Last update: {stats.get('last_update', 'Unknown')}")