claude-self-reflect 3.2.4 → 3.3.1

package/scripts/add-timestamp-indexes.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""Add timestamp indexes to all collections for OrderBy support."""
+
+import asyncio
+import os
+from pathlib import Path
+import sys
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from qdrant_client import AsyncQdrantClient
+from qdrant_client.models import PayloadSchemaType, OrderBy
+
+QDRANT_URL = os.getenv("QDRANT_URL", "http://localhost:6333")
+
+async def add_timestamp_indexes():
+    """Add timestamp indexes to all collections that need them."""
+    client = AsyncQdrantClient(url=QDRANT_URL)
+    
+    print("Adding timestamp indexes for temporal query support...")
+    print("="*60)
+    
+    # Get all collections
+    collections = await client.get_collections()
+    total = len(collections.collections)
+    print(f"Found {total} collections")
+    
+    success_count = 0
+    skip_count = 0
+    error_count = 0
+    
+    for i, col in enumerate(collections.collections, 1):
+        col_name = col.name
+        print(f"\n[{i}/{total}] Processing {col_name}...")
+        
+        try:
+            # Check if collection has points
+            info = await client.get_collection(col_name)
+            if info.points_count == 0:
+                print(f"  ⏭️  Skipped (empty collection)")
+                skip_count += 1
+                continue
+            
+            # Check if timestamp field exists
+            points, _ = await client.scroll(
+                collection_name=col_name,
+                limit=1,
+                with_payload=["timestamp"]
+            )
+            
+            if not points or not points[0].payload.get('timestamp'):
+                print(f"  ⏭️  Skipped (no timestamp field)")
+                skip_count += 1
+                continue
+            
+            # Try to use OrderBy to check if index exists
+            try:
+                await client.scroll(
+                    collection_name=col_name,
+                    order_by=OrderBy(key="timestamp", direction="desc"),
+                    limit=1
+                )
+                print(f"  ✅ Already has timestamp index")
+                skip_count += 1
+            except Exception as e:
+                if "No range index" in str(e):
+                    # Need to create index
+                    print(f"  🔧 Creating timestamp index...")
+                    try:
+                        await client.create_payload_index(
+                            collection_name=col_name,
+                            field_name="timestamp",
+                            field_schema=PayloadSchemaType.DATETIME
+                        )
+                        print(f"  ✅ Index created successfully")
+                        success_count += 1
+                    except Exception as create_error:
+                        print(f"  ❌ Failed to create index: {create_error}")
+                        error_count += 1
+                else:
+                    print(f"  ⚠️  Unexpected error: {e}")
+                    error_count += 1
+                    
+        except Exception as e:
+            print(f"  ❌ Error: {e}")
+            error_count += 1
+    
+    print("\n" + "="*60)
+    print("SUMMARY")
+    print("="*60)
+    print(f"✅ Indexes created: {success_count}")
+    print(f"⏭️  Skipped: {skip_count}")
+    print(f"❌ Errors: {error_count}")
+    print(f"📊 Total collections: {total}")
+    
+    # Verify temporal queries work
+    if success_count > 0:
+        print("\n" + "="*60)
+        print("VERIFYING TEMPORAL QUERIES")
+        print("="*60)
+        
+        # Find a collection with data to test
+        test_collection = None
+        for col in collections.collections:
+            try:
+                info = await client.get_collection(col.name)
+                if info.points_count > 100:  # Find one with decent amount of data
+                    test_collection = col.name
+                    break
+            except:
+                pass
+        
+        if test_collection:
+            print(f"Testing on {test_collection}...")
+            try:
+                # Test OrderBy
+                results, _ = await client.scroll(
+                    collection_name=test_collection,
+                    order_by=OrderBy(key="timestamp", direction="desc"),
+                    limit=3,
+                    with_payload=["timestamp", "text"]
+                )
+                
+                print(f"✅ OrderBy works! Found {len(results)} recent conversations:")
+                for r in results:
+                    ts = r.payload.get('timestamp', 'N/A')
+                    text = r.payload.get('text', '')[:60] + '...'
+                    print(f"  - {ts}: {text}")
+                    
+            except Exception as e:
+                print(f"❌ OrderBy test failed: {e}")
+
+if __name__ == "__main__":
+    asyncio.run(add_timestamp_indexes())

package/scripts/ast_grep_final_analyzer.py

@@ -0,0 +1,325 @@
+#!/usr/bin/env python3
+"""
+FINAL AST-GREP Analyzer with Unified Registry
+MANDATORY: Uses ast-grep-py + unified pattern registry
+NO regex fallbacks, NO simplifications
+"""
+
+import ast_grep_py as sg
+from pathlib import Path
+from typing import Dict, List, Any, Optional
+from datetime import datetime
+import json
+import sys
+
+# Import the unified registry
+sys.path.append(str(Path(__file__).parent))
+from ast_grep_unified_registry import get_unified_registry
+
+class FinalASTGrepAnalyzer:
+    """
+    Final production-ready AST-GREP analyzer.
+    MANDATORY components:
+    - ast-grep-py for AST matching
+    - Unified pattern registry (custom + catalog)
+    - NO regex patterns
+    - NO fallbacks
+    """
+
+    def __init__(self):
+        """Initialize with unified registry."""
+        self.registry = get_unified_registry()
+        all_patterns = self.registry.get_all_patterns()
+
+        print(f"✅ Loaded unified registry with {len(all_patterns)} patterns")
+        print(f"   Languages: Python, TypeScript, JavaScript")
+        print(f"   Good patterns: {len(self.registry.get_good_patterns())}")
+        print(f"   Bad patterns: {len(self.registry.get_bad_patterns())}")
+
+    def analyze_file(self, file_path: str) -> Dict[str, Any]:
+        """
+        Analyze a file using unified AST-GREP patterns.
+        Returns detailed quality metrics and pattern matches.
+        """
+        if not Path(file_path).exists():
+            raise FileNotFoundError(f"File not found: {file_path}")
+
+        # Detect language from file extension
+        language = self._detect_language(file_path)
+
+        with open(file_path, 'r', encoding='utf-8') as f:
+            content = f.read()
+
+        # Create SgRoot for the detected language
+        sg_language = self._get_sg_language(language)
+        root = sg.SgRoot(content, sg_language)
+        node = root.root()
+
+        # Get patterns for this language
+        language_patterns = self.registry.get_patterns_by_language(language)
+
+        # Track all matches
+        all_matches = []
+        pattern_errors = []
+        matches_by_category = {}
+
+        # Process each pattern
+        for pattern_def in language_patterns:
+            try:
+                pattern_str = pattern_def.get("pattern", "")
+                if not pattern_str:
+                    continue
+
+                # Find matches using ast-grep-py
+                matches = node.find_all(pattern=pattern_str)
+
+                if matches:
+                    category = pattern_def.get('category', 'unknown')
+                    if category not in matches_by_category:
+                        matches_by_category[category] = []
+
+                    match_info = {
+                        'category': category,
+                        'id': pattern_def['id'],
+                        'description': pattern_def.get('description', ''),
+                        'quality': pattern_def.get('quality', 'neutral'),
+                        'weight': pattern_def.get('weight', 0),
+                        'count': len(matches),
+                        'locations': [
+                            {
+                                'line': m.range().start.line + 1,
+                                'column': m.range().start.column,
+                                'text': m.text()[:80]
+                            } for m in matches[:5]  # First 5 examples
+                        ]
+                    }
+
+                    matches_by_category[category].append(match_info)
+                    all_matches.append(match_info)
+
+            except Exception as e:
+                # Record all pattern errors for debugging
+                pattern_errors.append({
+                    'pattern_id': pattern_def.get('id', '<unknown>'),
+                    'category': pattern_def.get('category', 'unknown'),
+                    'error': str(e)[:200]
+                })
+
+        # Calculate quality score
+        quality_score = self.registry.calculate_quality_score(all_matches)
+
+        # Count good vs bad patterns
+        good_matches = [m for m in all_matches if m['quality'] == 'good']
+        bad_matches = [m for m in all_matches if m['quality'] == 'bad']
+
+        good_count = sum(m['count'] for m in good_matches)
+        bad_count = sum(m['count'] for m in bad_matches)
+
+        return {
+            'file': file_path,
+            'timestamp': datetime.now().isoformat(),
+            'language': language,
+            'engine': 'ast-grep-py + unified registry',
+            'registry_info': {
+                'total_patterns_available': len(language_patterns),
+                'patterns_matched': len(all_matches),
+                'patterns_errored': len(pattern_errors),
+                'categories_found': list(matches_by_category.keys())
+            },
+            'matches_by_category': matches_by_category,
+            'all_matches': all_matches,
+            'errors': pattern_errors[:5],  # First 5 errors only
+            'quality_metrics': {
+                'quality_score': round(quality_score, 3),
+                'good_patterns_found': good_count,
+                'bad_patterns_found': bad_count,
+                'unique_patterns_matched': len(all_matches),
+                'total_issues': bad_count,
+                'total_good_practices': good_count
+            },
+            'recommendations': self._generate_recommendations(matches_by_category, quality_score)
+        }
+
+    def _detect_language(self, file_path: str) -> str:
+        """Detect language from file extension."""
+        ext = Path(file_path).suffix.lower()
+        lang_map = {
+            '.py': 'python',
+            '.ts': 'typescript',
+            '.tsx': 'tsx',
+            '.js': 'javascript',
+            '.jsx': 'jsx'
+        }
+        return lang_map.get(ext, 'python')
+
+    def _get_sg_language(self, language: str) -> str:
+        """Get ast-grep language identifier."""
+        # ast-grep-py uses different language identifiers
+        sg_map = {
+            'python': 'python',
+            'typescript': 'typescript',
+            'tsx': 'tsx',
+            'javascript': 'javascript',
+            'jsx': 'jsx'
+        }
+        return sg_map.get(language, 'python')
+
+    def _generate_recommendations(self, matches: Dict, score: float) -> List[str]:
+        """Generate actionable recommendations based on matches."""
+        recommendations = []
+
+        if score < 0.3:
+            recommendations.append("🔴 Critical: Code quality needs immediate attention")
+        elif score < 0.6:
+            recommendations.append("🟡 Warning: Several anti-patterns detected")
+        else:
+            recommendations.append("🟢 Good: Code follows most best practices")
+
+        # Check for specific issues
+        for category, category_matches in matches.items():
+            if 'antipatterns' in category:
+                total = sum(m['count'] for m in category_matches)
+                if total > 0:
+                    recommendations.append(f"Fix {total} anti-patterns in {category}")
+
+            if 'logging' in category:
+                prints = sum(m['count'] for m in category_matches if 'print' in m['id'])
+                if prints > 0:
+                    recommendations.append(f"Replace {prints} print statements with logger")
+
+            if 'error' in category:
+                bare = sum(m['count'] for m in category_matches if 'broad' in m['id'] or 'bare' in m['id'])
+                if bare > 0:
+                    recommendations.append(f"Fix {bare} bare except clauses")
+
+        return recommendations
+
+    def generate_report(self, result: Dict[str, Any]) -> str:
+        """Generate a comprehensive analysis report."""
+        report = []
+        report.append("# AST-GREP Pattern Analysis Report")
+        report.append(f"\n**File**: {result['file']}")
+        report.append(f"**Language**: {result['language']}")
+        report.append(f"**Timestamp**: {result['timestamp']}")
+        report.append(f"**Engine**: {result['engine']}")
+
+        # Quality overview
+        metrics = result['quality_metrics']
+        score = metrics['quality_score']
+        emoji = "🟢" if score > 0.7 else "🟡" if score > 0.4 else "🔴"
+
+        report.append("\n## Quality Overview")
+        report.append(f"- **Quality Score**: {emoji} {score:.1%}")
+        report.append(f"- **Good Practices**: {metrics['good_patterns_found']}")
+        report.append(f"- **Issues Found**: {metrics['total_issues']}")
+        report.append(f"- **Unique Patterns Matched**: {metrics['unique_patterns_matched']}")
+
+        # Recommendations
+        if result['recommendations']:
+            report.append("\n## Recommendations")
+            for rec in result['recommendations']:
+                report.append(f"- {rec}")
+
+        # Pattern matches by category
+        report.append("\n## Pattern Matches by Category")
+        for category, matches in result['matches_by_category'].items():
+            if matches:
+                total = sum(m['count'] for m in matches)
+                report.append(f"\n### {category} ({len(matches)} patterns, {total} matches)")
+
+                # Sort by count descending
+                sorted_matches = sorted(matches, key=lambda x: x['count'], reverse=True)
+
+                for match in sorted_matches[:5]:  # Top 5 per category
+                    quality_emoji = "✅" if match['quality'] == 'good' else "❌" if match['quality'] == 'bad' else "⚪"
+                    report.append(f"- {quality_emoji} **{match['id']}**: {match['count']} instances")
+                    report.append(f"  - {match['description']}")
+                    if match['locations']:
+                        loc = match['locations'][0]
+                        report.append(f"  - Example (line {loc['line']}): `{loc['text'][:50]}...`")
+
+        # Registry info
+        report.append("\n## Pattern Registry Statistics")
+        info = result['registry_info']
+        report.append(f"- **Patterns Available**: {info['total_patterns_available']}")
+        report.append(f"- **Patterns Matched**: {info['patterns_matched']}")
+        report.append(f"- **Categories Found**: {', '.join(info['categories_found'])}")
+
+        report.append("\n## Compliance")
+        report.append("✅ Using unified AST-GREP registry (custom + catalog)")
+        report.append("✅ Using ast-grep-py for AST matching")
+        report.append("✅ NO regex patterns or fallbacks")
+        report.append("✅ Production-ready pattern analysis")
+
+        return '\n'.join(report)
+
+
+def run_final_analysis():
+    """Run final AST-GREP analysis with unified registry."""
+    print("🚀 FINAL AST-GREP Analysis with Unified Registry")
+    print("=" * 60)
+
+    analyzer = FinalASTGrepAnalyzer()
+
+    # Analyze server.py
+    server_path = "/Users/ramakrishnanannaswamy/projects/claude-self-reflect/mcp-server/src/server.py"
+
+    print(f"\nAnalyzing: {server_path}")
+    print("-" * 40)
+
+    try:
+        result = analyzer.analyze_file(server_path)
+
+        # Display results
+        metrics = result['quality_metrics']
+        score = metrics['quality_score']
+
+        print(f"\n📊 Analysis Results:")
+        print(f"  Language: {result['language']}")
+        print(f"  Quality Score: {score:.1%}")
+        print(f"  Good Practices: {metrics['good_patterns_found']}")
+        print(f"  Issues: {metrics['total_issues']}")
+        print(f"  Patterns Matched: {metrics['unique_patterns_matched']}")
+
+        print(f"\n💡 Recommendations:")
+        for rec in result['recommendations']:
+            print(f"  {rec}")
+
+        # Top issues
+        bad_patterns = [m for m in result['all_matches'] if m['quality'] == 'bad']
+        if bad_patterns:
+            print(f"\n⚠️ Top Issues to Fix:")
+            sorted_bad = sorted(bad_patterns, key=lambda x: x['count'] * abs(x['weight']), reverse=True)
+            for pattern in sorted_bad[:5]:
+                print(f"  - {pattern['id']}: {pattern['count']} instances")
+                print(f"    {pattern['description']}")
+
+        # Generate and save report
+        report = analyzer.generate_report(result)
+        report_path = "/Users/ramakrishnanannaswamy/projects/claude-self-reflect/scripts/final_analysis_report.md"
+        with open(report_path, 'w') as f:
+            f.write(report)
+
+        print(f"\n📝 Full report saved to: {report_path}")
+
+        # Save JSON results
+        json_path = "/Users/ramakrishnanannaswamy/projects/claude-self-reflect/scripts/final_analysis_result.json"
+        with open(json_path, 'w') as f:
+            json.dump(result, f, indent=2)
+
+        print(f"📊 JSON results saved to: {json_path}")
+
+        print("\n✅ Final AST-GREP analysis complete!")
+        print("   - Unified registry with 41 patterns")
+        print("   - Support for Python, TypeScript, JavaScript")
+        print("   - Ready for production integration")
+
+        return result
+
+    except Exception as e:
+        print(f"\n❌ Analysis failed: {e}")
+        raise
+
+
+if __name__ == "__main__":
+    run_final_analysis()