claude-self-reflect 3.3.0 → 3.3.1

package/mcp-server/src/search_tools.py

@@ -4,6 +4,7 @@ import os
 import json
 import logging
 import time
+import html
 from typing import Optional, List, Dict, Any
 from datetime import datetime, timezone
 from pathlib import Path
@@ -168,20 +169,23 @@ class SearchTools:
                 if include_raw:
                     output += f"**Raw Payload:**\n```json\n{json.dumps(result.get('payload', {}), indent=2)}\n```\n\n"
         else:
-            # XML format (default)
-            output = f"<search_results>\n<query>{query}</query>\n<count>{len(results)}</count>\n"
+            # XML format (default) with proper escaping
+            def _esc(x): return html.escape(str(x), quote=False)
+
+            output = f"<search_results>\n<query>{_esc(query)}</query>\n<count>{len(results)}</count>\n"
             for i, result in enumerate(results, 1):
                 output += f"<result index=\"{i}\">\n"
                 output += f"  <score>{result['score']:.3f}</score>\n"
-                output += f"  <timestamp>{result.get('timestamp', 'N/A')}</timestamp>\n"
-                output += f"  <conversation_id>{result.get('conversation_id', 'N/A')}</conversation_id>\n"
+                output += f"  <timestamp>{_esc(result.get('timestamp', 'N/A'))}</timestamp>\n"
+                output += f"  <conversation_id>{_esc(result.get('conversation_id', 'N/A'))}</conversation_id>\n"
                 if not brief:
                     # Handle both 'content' and 'excerpt' fields
-                    content = result.get('content', result.get('excerpt', ''))
+                    content = result.get('content', result.get('excerpt', result.get('text', '')))
                     truncated = content[:500] + ('...' if len(content) > 500 else '')
                     output += f"  <content><![CDATA[{truncated}]]></content>\n"
                 if include_raw:
-                    output += f"  <raw_payload>{json.dumps(result.get('payload', {}))}</raw_payload>\n"
+                    # Use CDATA for large JSON payloads
+                    output += f"  <raw_payload><![CDATA[{json.dumps(result.get('payload', {}), ensure_ascii=False)}]]></raw_payload>\n"
                 output += "</result>\n"
             output += "</search_results>"
         
@@ -394,22 +398,29 @@ class SearchTools:
             top_result = max(all_results, key=lambda x: x.get('score', 0)) if all_results else None
             top_score = top_result.get('score', 0) if top_result else 0
             
-            # Format quick search response
+            # Format quick search response with proper XML escaping
+            def _esc(x): return html.escape(str(x), quote=False)
+
             if not top_result:
                 return "<quick_search><count>0</count><message>No matches found</message></quick_search>"
-            
+
+            # Get preview text and ensure we have content fallbacks
+            preview_text = top_result.get('excerpt', top_result.get('content', top_result.get('text', '')))[:200]
+
             return f"""<quick_search>
-<count>{collections_with_matches} collections with matches</count>
+<count>{collections_with_matches}</count>
+<collections_with_matches>{collections_with_matches}</collections_with_matches>
 <top_result>
   <score>{top_result['score']:.3f}</score>
-  <timestamp>{top_result.get('timestamp', 'N/A')}</timestamp>
-  <preview>{top_result.get('excerpt', top_result.get('content', ''))[:200]}...</preview>
+  <timestamp>{_esc(top_result.get('timestamp', 'N/A'))}</timestamp>
+  <preview><![CDATA[{preview_text}...]]></preview>
 </top_result>
 </quick_search>"""
-            
+
         except Exception as e:
             logger.error(f"Quick search failed: {e}", exc_info=True)
-            return f"<quick_search><error>Quick search failed: {str(e)}</error></quick_search>"
+            def _esc(x): return html.escape(str(x), quote=False)
+            return f"<quick_search><error>Quick search failed: {_esc(str(e))}</error></quick_search>"
 
     async def search_summary(
         self,
@@ -606,46 +617,83 @@ class SearchTools:
         project: Optional[str] = None
     ) -> str:
         """Search for conversations that analyzed a specific file."""
-        
+
         await ctx.debug(f"Searching for file: {file_path}, project={project}")
-        
+
         try:
-            # Normalize file path
-            normalized_path = str(Path(file_path).resolve())
-            
+            # Create multiple path variants to match how paths are stored
+            # Import uses normalize_file_path which replaces /Users/ with ~/
+            path_variants = set()
+
+            # Original path
+            path_variants.add(file_path)
+
+            # Basename only
+            path_variants.add(os.path.basename(file_path))
+
+            # Try to resolve if it's a valid path
+            try:
+                resolved_path = str(Path(file_path).resolve())
+                path_variants.add(resolved_path)
+
+                # Convert resolved path to ~/ format (matching how import stores it)
+                home_dir = str(Path.home())
+                if resolved_path.startswith(home_dir):
+                    tilde_path = resolved_path.replace(home_dir, '~', 1)
+                    path_variants.add(tilde_path)
+
+                # Also try with /Users/ replaced by ~/
+                if '/Users/' in resolved_path:
+                    path_variants.add(resolved_path.replace('/Users/', '~/', 1))
+            except:
+                pass
+
+            # If path starts with ~, also try expanded version
+            if file_path.startswith('~'):
+                expanded = os.path.expanduser(file_path)
+                path_variants.add(expanded)
+
+            # Convert all to forward slashes for consistency
+            path_variants = {p.replace('\\', '/') for p in path_variants if p}
+
+            await ctx.debug(f"Searching with path variants: {list(path_variants)}")
+
             # Search for file mentions in metadata
             collections_response = await self.qdrant_client.get_collections()
             collections = collections_response.collections
-            
-            # Define async function to search a single collection
+
+            # Define async function to search a single collection using scroll
             async def search_collection(collection_name: str):
                 try:
-                    # Search by payload filter
-                    search_results = await self.qdrant_client.search(
+                    from qdrant_client import models
+
+                    # Use scroll with proper filter for metadata-only search
+                    results, _ = await self.qdrant_client.scroll(
                         collection_name=collection_name,
-                        query_vector=[0] * 384,  # Dummy vector for metadata search
-                        limit=limit,
-                        query_filter={
-                            "must": [
-                                {
-                                    "key": "files_analyzed",
-                                    "match": {"any": [normalized_path]}
-                                }
+                        scroll_filter=models.Filter(
+                            should=[
+                                models.FieldCondition(
+                                    key="files_analyzed",
+                                    match=models.MatchValue(value=path_variant)
+                                )
+                                for path_variant in path_variants
                             ]
-                        }
+                        ),
+                        limit=limit,
+                        with_payload=True
                     )
-                    
-                    results = []
-                    for result in search_results:
-                        results.append({
-                            'conversation_id': result.payload.get('conversation_id'),
-                            'timestamp': result.payload.get('timestamp'),
-                            'content': result.payload.get('content', ''),
-                            'files_analyzed': result.payload.get('files_analyzed', []),
-                            'score': result.score
+
+                    formatted_results = []
+                    for point in results:
+                        formatted_results.append({
+                            'conversation_id': point.payload.get('conversation_id'),
+                            'timestamp': point.payload.get('timestamp'),
+                            'content': point.payload.get('content', point.payload.get('text', '')),
+                            'files_analyzed': point.payload.get('files_analyzed', []),
+                            'score': 1.0  # No score in scroll, use 1.0 for found items
                         })
-                    return results
-                    
+                    return formatted_results
+
                 except Exception as e:
                     await ctx.debug(f"Error searching {collection_name}: {e}")
                     return []

package/mcp-server/src/temporal_tools.py

@@ -82,10 +82,17 @@ class TemporalTools:
             
             # Filter collections by project
             if target_project != 'all':
+                # Use asyncio.to_thread to avoid blocking the event loop
+                import asyncio
                 from qdrant_client import QdrantClient as SyncQdrantClient
-                sync_client = SyncQdrantClient(url=self.qdrant_url)
-                resolver = ProjectResolver(sync_client)
-                project_collections = resolver.find_collections_for_project(target_project)
+
+                def get_project_collections():
+                    sync_client = SyncQdrantClient(url=self.qdrant_url)
+                    resolver = ProjectResolver(sync_client)
+                    return resolver.find_collections_for_project(target_project)
+
+                # Run sync client in thread pool to avoid blocking
+                project_collections = await asyncio.to_thread(get_project_collections)
                 
                 if not project_collections:
                     normalized_name = self.normalize_project_name(target_project)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-self-reflect",
-  "version": "3.3.0",
+  "version": "3.3.1",
   "description": "Give Claude perfect memory of all your conversations - Installation wizard for Python MCP server",
   "keywords": [
     "claude",
@@ -35,6 +35,11 @@
   },
   "files": [
     "installer/*.js",
+    "scripts/csr-status",
+    "scripts/session_quality_tracker.py",
+    "scripts/ast_grep_final_analyzer.py",
+    "scripts/ast_grep_unified_registry.py",
+    "scripts/update_patterns.py",
     "mcp-server/src/**/*.py",
     "mcp-server/pyproject.toml",
     "mcp-server/run-mcp.sh",

package/scripts/ast_grep_final_analyzer.py

@@ -0,0 +1,325 @@
+#!/usr/bin/env python3
+"""
+FINAL AST-GREP Analyzer with Unified Registry
+MANDATORY: Uses ast-grep-py + unified pattern registry
+NO regex fallbacks, NO simplifications
+"""
+
+import ast_grep_py as sg
+from pathlib import Path
+from typing import Dict, List, Any, Optional
+from datetime import datetime
+import json
+import sys
+
+# Import the unified registry
+sys.path.append(str(Path(__file__).parent))
+from ast_grep_unified_registry import get_unified_registry
+
+class FinalASTGrepAnalyzer:
+    """
+    Final production-ready AST-GREP analyzer.
+    MANDATORY components:
+    - ast-grep-py for AST matching
+    - Unified pattern registry (custom + catalog)
+    - NO regex patterns
+    - NO fallbacks
+    """
+
+    def __init__(self):
+        """Initialize with unified registry."""
+        self.registry = get_unified_registry()
+        all_patterns = self.registry.get_all_patterns()
+
+        print(f"✅ Loaded unified registry with {len(all_patterns)} patterns")
+        print(f"   Languages: Python, TypeScript, JavaScript")
+        print(f"   Good patterns: {len(self.registry.get_good_patterns())}")
+        print(f"   Bad patterns: {len(self.registry.get_bad_patterns())}")
+
+    def analyze_file(self, file_path: str) -> Dict[str, Any]:
+        """
+        Analyze a file using unified AST-GREP patterns.
+        Returns detailed quality metrics and pattern matches.
+        """
+        if not Path(file_path).exists():
+            raise FileNotFoundError(f"File not found: {file_path}")
+
+        # Detect language from file extension
+        language = self._detect_language(file_path)
+
+        with open(file_path, 'r', encoding='utf-8') as f:
+            content = f.read()
+
+        # Create SgRoot for the detected language
+        sg_language = self._get_sg_language(language)
+        root = sg.SgRoot(content, sg_language)
+        node = root.root()
+
+        # Get patterns for this language
+        language_patterns = self.registry.get_patterns_by_language(language)
+
+        # Track all matches
+        all_matches = []
+        pattern_errors = []
+        matches_by_category = {}
+
+        # Process each pattern
+        for pattern_def in language_patterns:
+            try:
+                pattern_str = pattern_def.get("pattern", "")
+                if not pattern_str:
+                    continue
+
+                # Find matches using ast-grep-py
+                matches = node.find_all(pattern=pattern_str)
+
+                if matches:
+                    category = pattern_def.get('category', 'unknown')
+                    if category not in matches_by_category:
+                        matches_by_category[category] = []
+
+                    match_info = {
+                        'category': category,
+                        'id': pattern_def['id'],
+                        'description': pattern_def.get('description', ''),
+                        'quality': pattern_def.get('quality', 'neutral'),
+                        'weight': pattern_def.get('weight', 0),
+                        'count': len(matches),
+                        'locations': [
+                            {
+                                'line': m.range().start.line + 1,
+                                'column': m.range().start.column,
+                                'text': m.text()[:80]
+                            } for m in matches[:5]  # First 5 examples
+                        ]
+                    }
+
+                    matches_by_category[category].append(match_info)
+                    all_matches.append(match_info)
+
+            except Exception as e:
+                # Record all pattern errors for debugging
+                pattern_errors.append({
+                    'pattern_id': pattern_def.get('id', '<unknown>'),
+                    'category': pattern_def.get('category', 'unknown'),
+                    'error': str(e)[:200]
+                })
+
+        # Calculate quality score
+        quality_score = self.registry.calculate_quality_score(all_matches)
+
+        # Count good vs bad patterns
+        good_matches = [m for m in all_matches if m['quality'] == 'good']
+        bad_matches = [m for m in all_matches if m['quality'] == 'bad']
+
+        good_count = sum(m['count'] for m in good_matches)
+        bad_count = sum(m['count'] for m in bad_matches)
+
+        return {
+            'file': file_path,
+            'timestamp': datetime.now().isoformat(),
+            'language': language,
+            'engine': 'ast-grep-py + unified registry',
+            'registry_info': {
+                'total_patterns_available': len(language_patterns),
+                'patterns_matched': len(all_matches),
+                'patterns_errored': len(pattern_errors),
+                'categories_found': list(matches_by_category.keys())
+            },
+            'matches_by_category': matches_by_category,
+            'all_matches': all_matches,
+            'errors': pattern_errors[:5],  # First 5 errors only
+            'quality_metrics': {
+                'quality_score': round(quality_score, 3),
+                'good_patterns_found': good_count,
+                'bad_patterns_found': bad_count,
+                'unique_patterns_matched': len(all_matches),
+                'total_issues': bad_count,
+                'total_good_practices': good_count
+            },
+            'recommendations': self._generate_recommendations(matches_by_category, quality_score)
+        }
+
+    def _detect_language(self, file_path: str) -> str:
+        """Detect language from file extension."""
+        ext = Path(file_path).suffix.lower()
+        lang_map = {
+            '.py': 'python',
+            '.ts': 'typescript',
+            '.tsx': 'tsx',
+            '.js': 'javascript',
+            '.jsx': 'jsx'
+        }
+        return lang_map.get(ext, 'python')
+
+    def _get_sg_language(self, language: str) -> str:
+        """Get ast-grep language identifier."""
+        # ast-grep-py uses different language identifiers
+        sg_map = {
+            'python': 'python',
+            'typescript': 'typescript',
+            'tsx': 'tsx',
+            'javascript': 'javascript',
+            'jsx': 'jsx'
+        }
+        return sg_map.get(language, 'python')
+
+    def _generate_recommendations(self, matches: Dict, score: float) -> List[str]:
+        """Generate actionable recommendations based on matches."""
+        recommendations = []
+
+        if score < 0.3:
+            recommendations.append("🔴 Critical: Code quality needs immediate attention")
+        elif score < 0.6:
+            recommendations.append("🟡 Warning: Several anti-patterns detected")
+        else:
+            recommendations.append("🟢 Good: Code follows most best practices")
+
+        # Check for specific issues
+        for category, category_matches in matches.items():
+            if 'antipatterns' in category:
+                total = sum(m['count'] for m in category_matches)
+                if total > 0:
+                    recommendations.append(f"Fix {total} anti-patterns in {category}")
+
+            if 'logging' in category:
+                prints = sum(m['count'] for m in category_matches if 'print' in m['id'])
+                if prints > 0:
+                    recommendations.append(f"Replace {prints} print statements with logger")
+
+            if 'error' in category:
+                bare = sum(m['count'] for m in category_matches if 'broad' in m['id'] or 'bare' in m['id'])
+                if bare > 0:
+                    recommendations.append(f"Fix {bare} bare except clauses")
+
+        return recommendations
+
+    def generate_report(self, result: Dict[str, Any]) -> str:
+        """Generate a comprehensive analysis report."""
+        report = []
+        report.append("# AST-GREP Pattern Analysis Report")
+        report.append(f"\n**File**: {result['file']}")
+        report.append(f"**Language**: {result['language']}")
+        report.append(f"**Timestamp**: {result['timestamp']}")
+        report.append(f"**Engine**: {result['engine']}")
+
+        # Quality overview
+        metrics = result['quality_metrics']
+        score = metrics['quality_score']
+        emoji = "🟢" if score > 0.7 else "🟡" if score > 0.4 else "🔴"
+
+        report.append("\n## Quality Overview")
+        report.append(f"- **Quality Score**: {emoji} {score:.1%}")
+        report.append(f"- **Good Practices**: {metrics['good_patterns_found']}")
+        report.append(f"- **Issues Found**: {metrics['total_issues']}")
+        report.append(f"- **Unique Patterns Matched**: {metrics['unique_patterns_matched']}")
+
+        # Recommendations
+        if result['recommendations']:
+            report.append("\n## Recommendations")
+            for rec in result['recommendations']:
+                report.append(f"- {rec}")
+
+        # Pattern matches by category
+        report.append("\n## Pattern Matches by Category")
+        for category, matches in result['matches_by_category'].items():
+            if matches:
+                total = sum(m['count'] for m in matches)
+                report.append(f"\n### {category} ({len(matches)} patterns, {total} matches)")
+
+                # Sort by count descending
+                sorted_matches = sorted(matches, key=lambda x: x['count'], reverse=True)
+
+                for match in sorted_matches[:5]:  # Top 5 per category
+                    quality_emoji = "✅" if match['quality'] == 'good' else "❌" if match['quality'] == 'bad' else "⚪"
+                    report.append(f"- {quality_emoji} **{match['id']}**: {match['count']} instances")
+                    report.append(f"  - {match['description']}")
+                    if match['locations']:
+                        loc = match['locations'][0]
+                        report.append(f"  - Example (line {loc['line']}): `{loc['text'][:50]}...`")
+
+        # Registry info
+        report.append("\n## Pattern Registry Statistics")
+        info = result['registry_info']
+        report.append(f"- **Patterns Available**: {info['total_patterns_available']}")
+        report.append(f"- **Patterns Matched**: {info['patterns_matched']}")
+        report.append(f"- **Categories Found**: {', '.join(info['categories_found'])}")
+
+        report.append("\n## Compliance")
+        report.append("✅ Using unified AST-GREP registry (custom + catalog)")
+        report.append("✅ Using ast-grep-py for AST matching")
+        report.append("✅ NO regex patterns or fallbacks")
+        report.append("✅ Production-ready pattern analysis")
+
+        return '\n'.join(report)
+
+
+def run_final_analysis():
+    """Run final AST-GREP analysis with unified registry."""
+    print("🚀 FINAL AST-GREP Analysis with Unified Registry")
+    print("=" * 60)
+
+    analyzer = FinalASTGrepAnalyzer()
+
+    # Analyze server.py
+    server_path = "/Users/ramakrishnanannaswamy/projects/claude-self-reflect/mcp-server/src/server.py"
+
+    print(f"\nAnalyzing: {server_path}")
+    print("-" * 40)
+
+    try:
+        result = analyzer.analyze_file(server_path)
+
+        # Display results
+        metrics = result['quality_metrics']
+        score = metrics['quality_score']
+
+        print(f"\n📊 Analysis Results:")
+        print(f"  Language: {result['language']}")
+        print(f"  Quality Score: {score:.1%}")
+        print(f"  Good Practices: {metrics['good_patterns_found']}")
+        print(f"  Issues: {metrics['total_issues']}")
+        print(f"  Patterns Matched: {metrics['unique_patterns_matched']}")
+
+        print(f"\n💡 Recommendations:")
+        for rec in result['recommendations']:
+            print(f"  {rec}")
+
+        # Top issues
+        bad_patterns = [m for m in result['all_matches'] if m['quality'] == 'bad']
+        if bad_patterns:
+            print(f"\n⚠️ Top Issues to Fix:")
+            sorted_bad = sorted(bad_patterns, key=lambda x: x['count'] * abs(x['weight']), reverse=True)
+            for pattern in sorted_bad[:5]:
+                print(f"  - {pattern['id']}: {pattern['count']} instances")
+                print(f"    {pattern['description']}")
+
+        # Generate and save report
+        report = analyzer.generate_report(result)
+        report_path = "/Users/ramakrishnanannaswamy/projects/claude-self-reflect/scripts/final_analysis_report.md"
+        with open(report_path, 'w') as f:
+            f.write(report)
+
+        print(f"\n📝 Full report saved to: {report_path}")
+
+        # Save JSON results
+        json_path = "/Users/ramakrishnanannaswamy/projects/claude-self-reflect/scripts/final_analysis_result.json"
+        with open(json_path, 'w') as f:
+            json.dump(result, f, indent=2)
+
+        print(f"📊 JSON results saved to: {json_path}")
+
+        print("\n✅ Final AST-GREP analysis complete!")
+        print("   - Unified registry with 41 patterns")
+        print("   - Support for Python, TypeScript, JavaScript")
+        print("   - Ready for production integration")
+
+        return result
+
+    except Exception as e:
+        print(f"\n❌ Analysis failed: {e}")
+        raise
+
+
+if __name__ == "__main__":
+    run_final_analysis()