claude-self-reflect 3.2.4 → 3.3.1

package/scripts/import-conversations-unified.py

@@ -13,11 +13,23 @@ import ast
 import re
 import fcntl
 import time
+import argparse
 from pathlib import Path
 from datetime import datetime
 from typing import List, Dict, Any, Optional, Set
 import logging
 
+# Load .env file if it exists
+try:
+    from dotenv import load_dotenv
+    # Load from project root
+    env_path = Path(__file__).parent.parent / '.env'
+    if env_path.exists():
+        load_dotenv(env_path)
+        print(f"Loaded .env from {env_path}")
+except ImportError:
+    pass  # dotenv not available, use system environment
+
 # Add the scripts directory to the Python path for utils import
 scripts_dir = Path(__file__).parent
 sys.path.insert(0, str(scripts_dir))
@@ -133,8 +145,17 @@ def ensure_collection(collection_name: str):
 
 def generate_embeddings(texts: List[str]) -> List[List[float]]:
     """Generate embeddings for texts."""
-    if PREFER_LOCAL_EMBEDDINGS or not VOYAGE_API_KEY:
-        embeddings = list(embedding_provider.passage_embed(texts))
+    # Use the global embedding_provider which gets updated by command-line args
+    if PREFER_LOCAL_EMBEDDINGS:
+        # FastEmbed uses 'embed' method, not 'passage_embed'
+        # Try 'embed' first, fall back to 'passage_embed' for compatibility
+        if hasattr(embedding_provider, 'embed'):
+            embeddings = list(embedding_provider.embed(texts))
+        elif hasattr(embedding_provider, 'passage_embed'):
+            # Fallback for older versions (shouldn't exist but kept for safety)
+            embeddings = list(embedding_provider.passage_embed(texts))
+        else:
+            raise AttributeError("FastEmbed provider has neither 'embed' nor 'passage_embed' method")
         return [emb.tolist() if hasattr(emb, 'tolist') else emb for emb in embeddings]
     else:
         response = embedding_provider.embed(texts, model="voyage-3")
@@ -355,7 +376,8 @@ def extract_metadata_single_pass(file_path: str) -> tuple[Dict[str, Any], str, i
                                                 # Extract code for AST analysis with bounds checking
                                                 if len(metadata['ast_elements']) < MAX_AST_ELEMENTS:
                                                     # Fix: More permissive regex to handle various fence formats
-                                                    code_blocks = re.findall(r'```[^`]*?\n(.*?)```', item.get('text', ''), re.DOTALL)
+                                                    # Handles both ```\n and ```python\n cases, with optional newline
+                                                    code_blocks = re.findall(r'```[^`\n]*\n?(.*?)```', item.get('text', ''), re.DOTALL)
                                                     for code_block in code_blocks[:MAX_CODE_BLOCKS]:  # Use defined constant
                                                         if len(metadata['ast_elements']) >= MAX_AST_ELEMENTS:
                                                             break
@@ -363,7 +385,11 @@ def extract_metadata_single_pass(file_path: str) -> tuple[Dict[str, Any], str, i
                                                         for elem in list(ast_elems)[:MAX_ELEMENTS_PER_BLOCK]:  # Use defined constant
                                                             if elem not in metadata['ast_elements'] and len(metadata['ast_elements']) < MAX_AST_ELEMENTS:
                                                                 metadata['ast_elements'].append(elem)
-                                        
+
+                                        elif item.get('type') == 'thinking':
+                                            # Also include thinking content in metadata extraction
+                                            text_content += item.get('thinking', '')
+
                                         elif item.get('type') == 'tool_use':
                                             tool_name = item.get('name', '')
                                             if tool_name and tool_name not in metadata['tools_used']:
@@ -410,39 +436,77 @@ def extract_metadata_single_pass(file_path: str) -> tuple[Dict[str, Any], str, i
     if all_text:
         combined_text = ' '.join(all_text[:MAX_CONCEPT_MESSAGES])  # Limit messages for concept extraction
         metadata['concepts'] = extract_concepts(combined_text)
-    
+
+    # MANDATORY: AST-GREP Pattern Analysis
+    # Analyze code quality for files mentioned in conversation
+    pattern_quality = {}
+    avg_quality_score = 0.0
+
+    try:
+        # Update patterns first (uses 24h cache, <100ms)
+        from update_patterns import check_and_update_patterns
+        check_and_update_patterns()
+
+        # Import analyzer
+        from ast_grep_final_analyzer import FinalASTGrepAnalyzer
+        analyzer = FinalASTGrepAnalyzer()
+
+        # Analyze edited and analyzed files
+        files_to_analyze = list(set(metadata['files_edited'] + metadata['files_analyzed'][:10]))
+        quality_scores = []
+
+        for file_path in files_to_analyze:
+            # Only analyze code files
+            if file_path and any(file_path.endswith(ext) for ext in ['.py', '.ts', '.js', '.tsx', '.jsx']):
+                try:
+                    # Check if file exists and is accessible
+                    if os.path.exists(file_path):
+                        result = analyzer.analyze_file(file_path)
+                        metrics = result['quality_metrics']
+                        pattern_quality[file_path] = {
+                            'score': metrics['quality_score'],
+                            'good_patterns': metrics['good_patterns_found'],
+                            'bad_patterns': metrics['bad_patterns_found'],
+                            'issues': metrics['total_issues']
+                        }
+                        quality_scores.append(metrics['quality_score'])
+                except Exception as e:
+                    logger.debug(f"Could not analyze {file_path}: {e}")
+
+        # Calculate average quality
+        if quality_scores:
+            avg_quality_score = sum(quality_scores) / len(quality_scores)
+
+    except Exception as e:
+        logger.debug(f"AST analysis not available: {e}")
+
+    # Add pattern analysis to metadata
+    metadata['pattern_analysis'] = pattern_quality
+    metadata['avg_quality_score'] = round(avg_quality_score, 3)
+
     # Set total messages
     metadata['total_messages'] = message_count
-    
+
     # Limit arrays
     metadata['files_analyzed'] = metadata['files_analyzed'][:MAX_FILES_ANALYZED]
     metadata['files_edited'] = metadata['files_edited'][:MAX_FILES_EDITED]
     metadata['tools_used'] = metadata['tools_used'][:MAX_TOOLS_USED]
     metadata['ast_elements'] = metadata['ast_elements'][:MAX_AST_ELEMENTS]
-    
+
     return metadata, first_timestamp or datetime.now().isoformat(), message_count
 
 def stream_import_file(jsonl_file: Path, collection_name: str, project_path: Path) -> int:
     """Stream import a single JSONL file without loading it into memory."""
     logger.info(f"Streaming import of {jsonl_file.name}")
-    
-    # Delete existing points for this conversation to prevent stale data
+
+    # Extract conversation ID
     conversation_id = jsonl_file.stem
-    try:
-        from qdrant_client.models import Filter, FieldCondition, MatchValue
-        client.delete(
-            collection_name=collection_name,
-            points_selector=Filter(
-                must=[FieldCondition(key="conversation_id", match=MatchValue(value=conversation_id))]
-            ),
-            wait=True
-        )
-        logger.info(f"Deleted existing points for conversation {conversation_id}")
-    except Exception as e:
-        logger.warning(f"Could not delete existing points for {conversation_id}: {e}")
-    
+
     # Extract metadata in first pass (lightweight)
     metadata, created_at, total_messages = extract_metadata_single_pass(str(jsonl_file))
+
+    # Track whether we should delete old points (only after successful import)
+    should_delete_old = False
     
     # Reset counters for each conversation (critical for correct indexing)
     current_message_index = 0  # Must be reset before processing each conversation
@@ -480,6 +544,11 @@ def stream_import_file(jsonl_file: Path, collection_name: str, project_path: Pat
                                         item_type = item.get('type', '')
                                         if item_type == 'text':
                                             text_parts.append(item.get('text', ''))
+                                        elif item_type == 'thinking':
+                                            # Include thinking content (from Claude's thinking blocks)
+                                            thinking_content = item.get('thinking', '')
+                                            if thinking_content:
+                                                text_parts.append(f"[Thinking] {thinking_content[:1000]}")  # Limit size
                                         elif item_type == 'tool_use':
                                             # Include tool use information
                                             tool_name = item.get('name', 'unknown')
@@ -581,10 +650,35 @@ def stream_import_file(jsonl_file: Path, collection_name: str, project_path: Pat
                 created_at, metadata, collection_name, project_path, total_messages
             )
             total_chunks += chunks
-        
+
+        # Only delete old points after successful import verification
+        if total_chunks > 0:
+            try:
+                from qdrant_client.models import Filter, FieldCondition, MatchValue
+                # Count old points before deletion for verification
+                old_count_filter = Filter(
+                    must=[FieldCondition(key="conversation_id", match=MatchValue(value=conversation_id))]
+                )
+                old_points = client.scroll(
+                    collection_name=collection_name,
+                    scroll_filter=old_count_filter,
+                    limit=1
+                )[0]
+
+                if len(old_points) > total_chunks + 5:  # Allow some tolerance
+                    # Only delete if we have significantly more old points than new
+                    client.delete(
+                        collection_name=collection_name,
+                        points_selector=old_count_filter,
+                        wait=True
+                    )
+                    logger.info(f"Deleted old points for conversation {conversation_id} after verifying new import")
+            except Exception as e:
+                logger.warning(f"Could not clean up old points for {conversation_id}: {e}")
+
         logger.info(f"Imported {total_chunks} chunks from {jsonl_file.name}")
         return total_chunks
-        
+
     except Exception as e:
         logger.error(f"Failed to import {jsonl_file}: {e}")
         return 0
@@ -673,6 +767,32 @@ def update_file_state(file_path: Path, state: dict, chunks: int):
 
 def main():
     """Main import function."""
+    # Parse command-line arguments
+    parser = argparse.ArgumentParser(description='Import conversations with unified embeddings support')
+    parser.add_argument('--prefer-voyage', action='store_true', 
+                       help='Use Voyage AI embeddings instead of local FastEmbed')
+    parser.add_argument('--limit', type=int, 
+                       help='Limit number of files to import')
+    parser.add_argument('--max-files-per-cycle', type=int, 
+                       help='Maximum files to process per cycle')
+    args = parser.parse_args()
+    
+    # Override environment variable if --prefer-voyage is specified
+    global PREFER_LOCAL_EMBEDDINGS, embedding_provider, embedding_dimension, collection_suffix
+    if args.prefer_voyage:
+        if not VOYAGE_API_KEY:
+            logger.error("--prefer-voyage specified but VOYAGE_KEY environment variable not set")
+            sys.exit(1)
+        logger.info("Command-line flag --prefer-voyage detected, switching to Voyage AI embeddings")
+        PREFER_LOCAL_EMBEDDINGS = False
+        
+        # Re-initialize embedding provider with Voyage
+        import voyageai
+        embedding_provider = voyageai.Client(api_key=VOYAGE_API_KEY)
+        embedding_dimension = 1024
+        collection_suffix = "voyage"
+        logger.info("Switched to Voyage AI embeddings (dimension: 1024)")
+    
     # Load state
     state = load_state()
     logger.info(f"Loaded state with {len(state.get('imported_files', {}))} previously imported files")
@@ -695,6 +815,7 @@ def main():
     logger.info(f"Found {len(project_dirs)} projects to import")
     
     total_imported = 0
+    files_processed = 0
     
     for project_dir in project_dirs:
         # Get collection name
@@ -707,13 +828,24 @@ def main():
         # Find JSONL files
         jsonl_files = sorted(project_dir.glob("*.jsonl"))
         
+        # Apply limit from command line if specified
+        if args.limit and files_processed >= args.limit:
+            logger.info(f"Reached limit of {args.limit} files, stopping import")
+            break
+            
         # Limit files per cycle if specified
-        max_files = int(os.getenv("MAX_FILES_PER_CYCLE", "1000"))
+        max_files = args.max_files_per_cycle or int(os.getenv("MAX_FILES_PER_CYCLE", "1000"))
         jsonl_files = jsonl_files[:max_files]
         
         for jsonl_file in jsonl_files:
+            # Check limit again per file
+            if args.limit and files_processed >= args.limit:
+                logger.info(f"Reached limit of {args.limit} files, stopping import")
+                break
+                
             if should_import_file(jsonl_file, state):
                 chunks = stream_import_file(jsonl_file, collection_name, project_dir)
+                files_processed += 1
                 if chunks > 0:
                     # Verify data is actually in Qdrant before marking as imported
                     from qdrant_client.models import Filter, FieldCondition, MatchValue

package/scripts/precompact-hook.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+# PreCompact hook for Claude Self-Reflect
+# Place this in ~/.claude/hooks/precompact or source it from there
+
+# Configuration
+CLAUDE_REFLECT_DIR="${CLAUDE_REFLECT_DIR:-$HOME/claude-self-reflect}"
+VENV_PATH="${VENV_PATH:-$CLAUDE_REFLECT_DIR/.venv}"
+IMPORT_TIMEOUT="${IMPORT_TIMEOUT:-30}"
+
+# Check if Claude Self-Reflect is installed
+if [ ! -d "$CLAUDE_REFLECT_DIR" ]; then
+    echo "Claude Self-Reflect not found at $CLAUDE_REFLECT_DIR" >&2
+    exit 0  # Exit gracefully
+fi
+
+# Check if virtual environment exists
+if [ ! -d "$VENV_PATH" ]; then
+    echo "Virtual environment not found at $VENV_PATH" >&2
+    exit 0  # Exit gracefully
+fi
+
+# Run quick import with timeout
+echo "Updating conversation memory..." >&2
+timeout $IMPORT_TIMEOUT bash -c "
+    source '$VENV_PATH/bin/activate' 2>/dev/null
+    python '$CLAUDE_REFLECT_DIR/scripts/import-latest.py' 2>&1 | \
+        grep -E '(Quick import completed|Imported|Warning)' >&2
+" || {
+    echo "Quick import timed out after ${IMPORT_TIMEOUT}s" >&2
+}
+
+# Always exit successfully to not block compacting
+exit 0