claude-self-reflect 3.3.1 → 4.0.1

package/scripts/import-conversations-unified.py

@@ -15,7 +15,7 @@ import fcntl
 import time
 import argparse
 from pathlib import Path
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import List, Dict, Any, Optional, Set
 import logging
 
@@ -34,6 +34,9 @@ except ImportError:
 scripts_dir = Path(__file__).parent
 sys.path.insert(0, str(scripts_dir))
 
+# Import UnifiedStateManager
+from unified_state_manager import UnifiedStateManager
+
 from qdrant_client import QdrantClient
 from qdrant_client.models import PointStruct, Distance, VectorParams
 
@@ -72,32 +75,15 @@ MAX_FILES_EDITED = 20
 MAX_TOOLS_USED = 15
 MAX_CONCEPT_MESSAGES = 50
 
-# Robust cross-platform state file resolution
-def get_default_state_file():
-    """Determine the default state file location with cross-platform support."""
-    from pathlib import Path
-    
-    # Check if we're in Docker (more reliable than just checking /config)
-    docker_indicators = [
-        Path("/.dockerenv").exists(),  # Docker creates this file
-        os.path.exists("/config") and os.access("/config", os.W_OK)  # Mounted config dir with write access
-    ]
-    
-    if any(docker_indicators):
-        return "/config/imported-files.json"
-    
-    # Use pathlib for cross-platform home directory path
-    home_state = Path.home() / ".claude-self-reflect" / "config" / "imported-files.json"
-    return str(home_state)
-
-# Get state file path with env override support
+# Initialize UnifiedStateManager
+# Support legacy STATE_FILE environment variable
 env_state = os.getenv("STATE_FILE")
 if env_state:
-    # Normalize any user-provided path to absolute
     from pathlib import Path
-    STATE_FILE = str(Path(env_state).expanduser().resolve())
+    state_file_path = Path(env_state).expanduser().resolve()
+    state_manager = UnifiedStateManager(state_file_path)
 else:
-    STATE_FILE = get_default_state_file()
+    state_manager = UnifiedStateManager()  # Uses default location
 PREFER_LOCAL_EMBEDDINGS = os.getenv("PREFER_LOCAL_EMBEDDINGS", "true").lower() == "true"
 VOYAGE_API_KEY = os.getenv("VOYAGE_KEY")
 MAX_CHUNK_SIZE = int(os.getenv("MAX_CHUNK_SIZE", "50"))  # Messages per chunk
@@ -335,22 +321,27 @@ def extract_metadata_single_pass(file_path: str) -> tuple[Dict[str, Any], str, i
         "concepts": [],
         "ast_elements": [],
         "has_code_blocks": False,
-        "total_messages": 0
+        "total_messages": 0,
+        "project_path": None  # Add project path from cwd
     }
-    
+
     first_timestamp = None
     message_count = 0
     all_text = []
-    
+
     try:
         with open(file_path, 'r', encoding='utf-8') as f:
             for line in f:
                 if not line.strip():
                     continue
-                    
+
                 try:
                     data = json.loads(line)
-                    
+
+                    # Extract cwd (current working directory) as project path
+                    if metadata["project_path"] is None and 'cwd' in data:
+                        metadata["project_path"] = data.get('cwd')
+
                     # Get timestamp from first valid entry
                     if first_timestamp is None and 'timestamp' in data:
                         first_timestamp = data.get('timestamp')
@@ -681,18 +672,13 @@ def stream_import_file(jsonl_file: Path, collection_name: str, project_path: Pat
 
     except Exception as e:
         logger.error(f"Failed to import {jsonl_file}: {e}")
+        # Mark file as failed in state manager
+        try:
+            state_manager.mark_file_failed(str(jsonl_file), str(e))
+        except Exception as state_error:
+            logger.warning(f"Could not mark file as failed in state: {state_error}")
         return 0
 
-def _locked_open(path, mode):
-    """Open file with exclusive lock for concurrent safety."""
-    f = open(path, mode)
-    try:
-        fcntl.flock(f.fileno(), fcntl.LOCK_EX)
-    except Exception:
-        f.close()
-        raise
-    return f
-
 def _with_retries(fn, attempts=3, base_sleep=0.5):
     """Execute function with retries and exponential backoff."""
     for i in range(attempts):
@@ -704,66 +690,78 @@ def _with_retries(fn, attempts=3, base_sleep=0.5):
             time.sleep(base_sleep * (2 ** i))
             logger.debug(f"Retrying after error: {e}")
 
-def load_state() -> dict:
-    """Load import state with file locking."""
-    if os.path.exists(STATE_FILE):
-        try:
-            with _locked_open(STATE_FILE, 'r') as f:
-                return json.load(f)
-        except Exception as e:
-            logger.warning(f"Failed to load state: {e}")
-    return {"imported_files": {}}
-
-def save_state(state: dict):
-    """Save import state with atomic write."""
-    # Fix: Handle case where STATE_FILE has no directory component
-    state_dir = os.path.dirname(STATE_FILE)
-    if state_dir:
-        os.makedirs(state_dir, exist_ok=True)
-    
-    # Use atomic write with locking to prevent corruption
-    temp_file = f"{STATE_FILE}.tmp"
-    with _locked_open(temp_file, 'w') as f:
-        json.dump(state, f, indent=2)
-        f.flush()
-        os.fsync(f.fileno())
-    
-    # Atomic rename (on POSIX systems)
-    os.replace(temp_file, STATE_FILE)
-
-def should_import_file(file_path: Path, state: dict) -> bool:
-    """Check if file should be imported."""
-    file_str = str(file_path)
-    if file_str in state.get("imported_files", {}):
-        file_info = state["imported_files"][file_str]
-        last_modified = file_path.stat().st_mtime
-        
-        # Check if file has been modified
-        if file_info.get("last_modified") != last_modified:
-            logger.info(f"File modified, will re-import: {file_path.name}")
-            return True
-            
-        # Check for suspiciously low chunk counts (likely failed imports)
-        chunks = file_info.get("chunks", 0)
-        file_size_kb = file_path.stat().st_size / 1024
-        
-        # Heuristic: Files > 10KB should have more than 2 chunks
-        if file_size_kb > 10 and chunks <= 2:
-            logger.warning(f"File has suspiciously low chunks ({chunks}) for size {file_size_kb:.1f}KB, will re-import: {file_path.name}")
-            return True
-            
-        logger.info(f"Skipping unchanged file: {file_path.name}")
-        return False
-    return True
-
-def update_file_state(file_path: Path, state: dict, chunks: int):
-    """Update state for imported file."""
-    file_str = str(file_path)
-    state["imported_files"][file_str] = {
-        "imported_at": datetime.now().isoformat(),
-        "last_modified": file_path.stat().st_mtime,
-        "chunks": chunks
-    }
+def should_import_file(file_path: Path) -> bool:
+    """Check if file should be imported using UnifiedStateManager."""
+    try:
+        # Get imported files from state manager
+        imported_files = state_manager.get_imported_files()
+
+        # Normalize the file path for comparison
+        normalized_path = state_manager.normalize_path(str(file_path))
+
+        if normalized_path in imported_files:
+            file_info = imported_files[normalized_path]
+
+            # Skip if file failed and we haven't reached retry limit
+            if file_info.get("status") == "failed" and file_info.get("retry_count", 0) >= 3:
+                logger.info(f"Skipping failed file (max retries reached): {file_path.name}")
+                return False
+
+            # Get file modification time for comparison
+            last_modified = file_path.stat().st_mtime
+            stored_modified = file_info.get("last_modified")
+
+            # Check if file has been modified (convert stored timestamp to float if needed)
+            if stored_modified:
+                try:
+                    # Parse ISO timestamp to float for comparison
+                    stored_time = datetime.fromisoformat(stored_modified.replace("Z", "+00:00")).timestamp()
+                    if abs(last_modified - stored_time) > 1:  # Allow 1 second tolerance
+                        logger.info(f"File modified, will re-import: {file_path.name}")
+                        return True
+                except (ValueError, TypeError):
+                    # If we can't parse the stored time, re-import to be safe
+                    logger.warning(f"Could not parse stored modification time, will re-import: {file_path.name}")
+                    return True
+
+            # Check for suspiciously low chunk counts (likely failed imports)
+            chunks = file_info.get("chunks", 0)
+            file_size_kb = file_path.stat().st_size / 1024
+
+            # Heuristic: Files > 10KB should have more than 2 chunks
+            if file_size_kb > 10 and chunks <= 2 and file_info.get("status") != "failed":
+                logger.warning(f"File has suspiciously low chunks ({chunks}) for size {file_size_kb:.1f}KB, will re-import: {file_path.name}")
+                return True
+
+            # Skip if successfully imported
+            if file_info.get("status") == "completed":
+                logger.info(f"Skipping successfully imported file: {file_path.name}")
+                return False
+
+        return True
+
+    except Exception as e:
+        logger.warning(f"Error checking import status for {file_path}: {e}")
+        return True  # Default to importing if we can't check status
+
+def update_file_state(file_path: Path, chunks: int, collection_name: str):
+    """Update state for imported file using UnifiedStateManager."""
+    try:
+        # Determine embedding mode from collection suffix
+        embedding_mode = "local" if collection_suffix == "local" else "cloud"
+
+        # Add file to state manager
+        state_manager.add_imported_file(
+            file_path=str(file_path),
+            chunks=chunks,
+            importer="streaming",
+            collection=collection_name,
+            embedding_mode=embedding_mode,
+            status="completed"
+        )
+        logger.debug(f"Updated state for {file_path.name}: {chunks} chunks")
+    except Exception as e:
+        logger.error(f"Failed to update state for {file_path}: {e}")
 
 def main():
     """Main import function."""
@@ -793,9 +791,9 @@ def main():
         collection_suffix = "voyage"
         logger.info("Switched to Voyage AI embeddings (dimension: 1024)")
     
-    # Load state
-    state = load_state()
-    logger.info(f"Loaded state with {len(state.get('imported_files', {}))} previously imported files")
+    # Get status from state manager
+    status = state_manager.get_status()
+    logger.info(f"Loaded state with {status['indexed_files']} previously imported files")
     
     # Find all projects
     # Use LOGS_DIR env var, or fall back to Claude projects directory, then /logs for Docker
@@ -843,7 +841,7 @@ def main():
                 logger.info(f"Reached limit of {args.limit} files, stopping import")
                 break
                 
-            if should_import_file(jsonl_file, state):
+            if should_import_file(jsonl_file):
                 chunks = stream_import_file(jsonl_file, collection_name, project_dir)
                 files_processed += 1
                 if chunks > 0:
@@ -863,8 +861,7 @@ def main():
                         
                         if actual_count > 0:
                             logger.info(f"Verified {actual_count} points in Qdrant for {conversation_id}")
-                            update_file_state(jsonl_file, state, chunks)
-                            save_state(state)
+                            update_file_state(jsonl_file, chunks, collection_name)
                             total_imported += 1
                         else:
                             logger.error(f"No points found in Qdrant for {conversation_id} despite {chunks} chunks processed - not marking as imported")
@@ -878,6 +875,11 @@ def main():
                     # Critical fix: Don't mark files with 0 chunks as imported
                     # This allows retry on next run
                     logger.warning(f"File produced 0 chunks, not marking as imported: {jsonl_file.name}")
+                    # Mark as failed so we don't keep retrying indefinitely
+                    try:
+                        state_manager.mark_file_failed(str(jsonl_file), "File produced 0 chunks during import")
+                    except Exception as state_error:
+                        logger.warning(f"Could not mark file as failed in state: {state_error}")
     
     logger.info(f"Import complete: processed {total_imported} files")
 

package/scripts/session_quality_tracker.py

@@ -106,7 +106,7 @@ class SessionQualityTracker:
 
         return edited_files
 
-    def analyze_session_quality(self, session_file: Optional[Path] = None) -> Dict[str, Any]:
+    def analyze_session_quality(self, session_file: Optional[Path] = None, use_tracker: bool = False) -> Dict[str, Any]:
         """
         Analyze code quality for all files edited in current session.
         Returns quality report with actionable insights.
@@ -114,6 +114,22 @@ class SessionQualityTracker:
         # Update patterns (uses cache, <100ms)
         check_and_update_patterns()
 
+        # Check for session edit tracker first (priority mode)
+        if use_tracker or (not session_file):
+            tracker_file = Path.home() / ".claude-self-reflect" / "current_session_edits.json"
+            if tracker_file.exists():
+                try:
+                    with open(tracker_file, 'r') as f:
+                        tracker_data = json.load(f)
+                        edited_files = set(tracker_data.get('edited_files', []))
+                        if edited_files:
+                            logger.info(f"Using session tracker: {len(edited_files)} files edited in session")
+                            self.current_session_id = 'active_session'
+                            # Use Session scope label for tracked edits
+                            return self._analyze_files_with_scope(edited_files, scope_label='Session')
+                except Exception as e:
+                    logger.debug(f"Error reading tracker file: {e}")
+
         # Find active session if not provided
         if not session_file:
             session_file = self.find_active_session()
@@ -188,7 +204,7 @@ class SessionQualityTracker:
                 'avg_quality_score': round(avg_quality, 3),
                 'total_issues': total_issues,
                 'total_good_patterns': total_good_patterns,
-                'quality_grade': self._get_quality_grade(avg_quality)
+                'quality_grade': self._get_quality_grade(avg_quality, total_issues)
             },
             'file_reports': file_reports,
             'actionable_items': self._generate_actionable_items(file_reports),
@@ -212,20 +228,92 @@ class SessionQualityTracker:
         top_issues.sort(key=lambda x: x['count'], reverse=True)
         return top_issues[:5]  # Top 5 issues
 
-    def _get_quality_grade(self, score: float) -> str:
-        """Convert quality score to letter grade."""
-        if score >= 0.9:
-            return 'A+'
-        elif score >= 0.8:
-            return 'A'
-        elif score >= 0.7:
-            return 'B'
-        elif score >= 0.6:
-            return 'C'
-        elif score >= 0.5:
-            return 'D'
-        else:
+    def _get_quality_grade(self, score: float, total_issues: int = 0) -> str:
+        """
+        Convert quality score to letter grade.
+        Based on consensus: issues should dominate grading.
+
+        Grade boundaries (adjusted for issue count):
+        - A+: score >= 0.97 AND issues <= 5
+        - A:  score >= 0.93 AND issues <= 20
+        - B:  score >= 0.83 AND issues <= 50
+        - C:  score >= 0.73 AND issues <= 100
+        - D:  score >= 0.60
+        - F:  score < 0.60
+        """
+        # Hard caps based on issue count (industry standard)
+        if total_issues > 200:
             return 'F'
+        elif total_issues > 100:
+            # Many issues - max grade is C
+            if score >= 0.77:
+                return 'C+'
+            elif score >= 0.73:
+                return 'C'
+            elif score >= 0.70:
+                return 'C-'
+            elif score >= 0.60:
+                return 'D'
+            else:
+                return 'F'
+        elif total_issues > 50:
+            # Moderate issues - max grade is B
+            if score >= 0.87:
+                return 'B+'
+            elif score >= 0.83:
+                return 'B'
+            elif score >= 0.80:
+                return 'B-'
+            elif score >= 0.73:
+                return 'C'
+            elif score >= 0.60:
+                return 'D'
+            else:
+                return 'F'
+        elif total_issues > 20:
+            # Some issues - max grade is A-
+            if score >= 0.90:
+                return 'A-'
+            elif score >= 0.87:
+                return 'B+'
+            elif score >= 0.83:
+                return 'B'
+            elif score >= 0.73:
+                return 'C'
+            elif score >= 0.60:
+                return 'D'
+            else:
+                return 'F'
+        elif total_issues > 5:
+            # Few issues - max grade is A
+            if score >= 0.93:
+                return 'A'
+            elif score >= 0.90:
+                return 'A-'
+            elif score >= 0.83:
+                return 'B'
+            elif score >= 0.73:
+                return 'C'
+            elif score >= 0.60:
+                return 'D'
+            else:
+                return 'F'
+        else:
+            # Very few issues (0-5) - can achieve A+
+            if score >= 0.97:
+                return 'A+'
+            elif score >= 0.93:
+                return 'A'
+            elif score >= 0.90:
+                return 'A-'
+            elif score >= 0.83:
+                return 'B'
+            elif score >= 0.73:
+                return 'C'
+            elif score >= 0.60:
+                return 'D'
+            else:
+                return 'F'
 
     def _generate_actionable_items(self, file_reports: Dict) -> List[str]:
         """Generate actionable recommendations for the user."""
@@ -263,23 +351,101 @@ class SessionQualityTracker:
 
         return actions
 
+    def _analyze_files_with_scope(self, edited_files: set, scope_label: str = 'Session') -> Dict[str, Any]:
+        """
+        Analyze specific files with a given scope label.
+        Used for both session tracking and fallback modes.
+        """
+        # Analyze each edited file
+        file_reports = {}
+        total_issues = 0
+        total_good_patterns = 0
+        quality_scores = []
+
+        for file_path in edited_files:
+            # Only analyze code files
+            if any(str(file_path).endswith(ext) for ext in ['.py', '.ts', '.js', '.tsx', '.jsx']):
+                try:
+                    result = self.analyzer.analyze_file(file_path)
+                    metrics = result['quality_metrics']
+
+                    file_reports[file_path] = {
+                        'quality_score': metrics['quality_score'],
+                        'good_patterns': metrics['good_patterns_found'],
+                        'issues': metrics['total_issues'],
+                        'recommendations': result.get('recommendations', [])[:3],  # Top 3
+                        'top_issues': self._get_top_issues(result)
+                    }
+
+                    total_issues += metrics['total_issues']
+                    total_good_patterns += metrics['good_patterns_found']
+                    quality_scores.append(metrics['quality_score'])
+
+                    # Track quality history
+                    if file_path not in self.quality_history:
+                        self.quality_history[file_path] = []
+                    self.quality_history[file_path].append({
+                        'timestamp': datetime.now().isoformat(),
+                        'score': metrics['quality_score']
+                    })
+
+                except Exception as e:
+                    logger.error(f"Failed to analyze {file_path}: {e}")
+
+        if not file_reports:
+            return {
+                'status': 'no_code_files',
+                'session_id': self.current_session_id,
+                'scope_label': scope_label,
+                'message': 'No analyzable code files in session'
+            }
+
+        # Calculate session average
+        avg_quality = sum(quality_scores) / len(quality_scores) if quality_scores else 0
+
+        # Generate session report
+        return {
+            'status': 'success',
+            'session_id': self.current_session_id,
+            'scope_label': scope_label,  # Use provided scope label
+            'timestamp': datetime.now().isoformat(),
+            'summary': {
+                'files_analyzed': len(file_reports),
+                'avg_quality_score': round(avg_quality, 3),
+                'total_issues': total_issues,
+                'total_good_patterns': total_good_patterns,
+                'quality_grade': self._get_quality_grade(avg_quality, total_issues)
+            },
+            'file_reports': file_reports,
+            'actionable_items': self._generate_actionable_items(file_reports),
+            'quality_trend': self._calculate_quality_trend()
+        }
+
     def analyze_recent_files(self) -> Dict[str, Any]:
         """Analyze core project files when no session is found."""
-        project_root = Path(__file__).parent.parent
-
-        # Define core project files to analyze (not test files)
-        core_files = [
-            "scripts/session_quality_tracker.py",
-            "scripts/cc-statusline-unified.py",
-            "scripts/pattern_registry_enhanced.py",
-            "scripts/simplified_metadata_extractor.py",
-            "scripts/streaming-watcher.py",
-            "scripts/quality-report.py",
-            "mcp-server/src/server.py",
-            "mcp-server/src/search_tools.py",
-            "mcp-server/src/temporal_tools.py",
-            "mcp-server/src/reflection_tools.py",
-        ]
+        # Use current working directory as project root
+        project_root = Path.cwd()
+
+        # Find code files in the project dynamically
+        code_extensions = {'.py', '.js', '.ts', '.jsx', '.tsx', '.java', '.cpp', '.c',
+                          '.h', '.hpp', '.rs', '.go', '.rb', '.php'}
+
+        core_files = []
+        # Look for code files in the project (limit to avoid too many files)
+        for ext in code_extensions:
+            files = list(project_root.rglob(f'*{ext}'))
+            # Filter out common non-source directories
+            files = [f for f in files if not any(
+                skip in f.parts for skip in ['venv', '.venv', 'node_modules', '.git',
+                                            '__pycache__', '.pytest_cache', 'dist',
+                                            'build', 'target', '.idea', '.vscode']
+            )]
+            core_files.extend(files[:20])  # Take up to 20 files per extension
+            if len(core_files) >= 50:  # Increased limit to 50 files for better coverage
+                break
+
+        # Convert to relative paths
+        core_files = [str(f.relative_to(project_root)) for f in core_files[:50]]
 
         edited_files = set()
         for file_path in core_files:
@@ -289,10 +455,7 @@ class SessionQualityTracker:
 
         # Also check for recently modified files (last 30 minutes) to catch actual work
         try:
-            # Validate project_root is within expected bounds
-            if not str(project_root.resolve()).startswith(str(Path(__file__).parent.parent.resolve())):
-                logger.error("Security: Invalid project root path")
-                return {}
+            # No need to validate project_root - we can analyze any project
 
             # Use pathlib instead of subprocess for safer file discovery
             scripts_dir = project_root / "scripts"
@@ -321,7 +484,7 @@ class SessionQualityTracker:
         total_good_patterns = 0
         quality_scores = []
 
-        for file_path in list(edited_files)[:10]:  # Limit to 10 files for performance
+        for file_path in list(edited_files)[:50]:  # Analyze up to 50 files for better coverage
             try:
                 result = self.analyzer.analyze_file(file_path)
                 metrics = result['quality_metrics']
@@ -367,7 +530,7 @@ class SessionQualityTracker:
                 'avg_quality_score': round(avg_quality, 3),
                 'total_issues': total_issues,
                 'total_good_patterns': total_good_patterns,
-                'quality_grade': self._get_quality_grade(avg_quality)
+                'quality_grade': self._get_quality_grade(avg_quality, total_issues)
             },
             'file_reports': file_reports,
             'actionable_items': self._generate_actionable_items(file_reports),
@@ -441,14 +604,14 @@ class SessionQualityTracker:
         return '\n'.join(report)
 
 
-def main():
+def main(use_tracker=False):
     """Run session quality analysis."""
     tracker = SessionQualityTracker()
 
     logger.info("🔍 Analyzing current session code quality...")
     logger.info("")
 
-    analysis = tracker.analyze_session_quality()
+    analysis = tracker.analyze_session_quality(use_tracker=use_tracker)
     report = tracker.generate_report(analysis)
 
     logger.info(report)
@@ -456,8 +619,8 @@ def main():
     # Save report for watcher integration - PER PROJECT
     # Always save cache, even with fallback analysis
     if analysis.get('status') in ['success', 'fallback']:
-        # Get project name from current directory
-        project_name = os.path.basename(os.getcwd())
+        # Get project name from environment or current directory
+        project_name = os.environ.get('QUALITY_PROJECT_NAME', os.path.basename(os.getcwd()))
         # Secure sanitization with whitelist approach
         import re
         safe_project_name = re.sub(r'[^a-zA-Z0-9_-]', '_', project_name)[:100]
@@ -478,4 +641,21 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    import argparse
+    parser = argparse.ArgumentParser(description='Analyze code quality for projects')
+    parser.add_argument('--project-path', help='Path to the project to analyze')
+    parser.add_argument('--project-name', help='Name of the project for cache file')
+    parser.add_argument('--use-tracker', action='store_true',
+                        help='Use session edit tracker for analysis')
+    args = parser.parse_args()
+
+    # If external project specified, change to that directory
+    if args.project_path:
+        os.chdir(args.project_path)
+
+    # Override project name if specified
+    if args.project_name:
+        # This will be used in the main() function for cache naming
+        os.environ['QUALITY_PROJECT_NAME'] = args.project_name
+
+    main(use_tracker=args.use_tracker)