claude-self-reflect 2.4.8 → 2.4.10

package/.claude/agents/reflection-specialist.md

@@ -770,4 +770,48 @@ After importing:
 4. **Restart MCP After Import**: Ensures new collections are recognized
 5. **Verify with Search**: Test that new content is searchable
 
+## Quick Import for Current Project (NEW in v2.4.8)
+
+For rapid updates when working on a single project, use the optimized quick import:
+
+### Quick Import Script
+```bash
+# Import only recent conversations (last 2 hours by default)
+cd /path/to/your/project
+source ~/claude-self-reflect/venv/bin/activate
+python ~/claude-self-reflect/scripts/import-latest.py
+
+# Customize time window
+export IMPORT_HOURS_BACK=4  # Import last 4 hours
+python ~/claude-self-reflect/scripts/import-latest.py
+```
+
+### PreCompact Hook Integration
+To automatically update conversations before compacting:
+
+```bash
+# Install the hook (one-time setup)
+cp ~/claude-self-reflect/scripts/precompact-hook.sh ~/.claude/hooks/precompact
+# Or source it from your existing precompact hook:
+echo "source ~/claude-self-reflect/scripts/precompact-hook.sh" >> ~/.claude/hooks/precompact
+```
+
+### Performance Expectations
+- **Full import**: 2-7 minutes (all projects, all history)
+- **Quick import**: 30-60 seconds (current project, recent files only)
+- **Target**: <10 seconds (future optimization)
+
+### When to Use Quick Import
+- Before starting a new Claude session
+- After significant conversation progress  
+- Via PreCompact hook (automatic)
+- When recent conversations aren't in search results
+
+### Troubleshooting Quick Import
+If quick import fails:
+1. Ensure you're in a project directory with Claude logs
+2. Check virtual environment is activated
+3. Verify project has a collection: `python scripts/check-collections.py`
+4. For first-time projects, run full import once
+
 Remember: You're not just a search tool - you're a memory augmentation system that helps maintain continuity, prevent repeated work, and leverage collective knowledge across all Claude conversations.

package/README.md

@@ -26,6 +26,15 @@ Your conversations become searchable. Your decisions stay remembered. Your conte
 - **Node.js** 16+ (for the setup wizard)
 - **Claude Desktop** app
 
+## System Requirements
+
+### Memory
+- **Docker Memory**: 2GB minimum (4GB recommended for initial setup)
+- **First Import**: May take 2-7 minutes to process all conversations
+- **Subsequent Imports**: <60 seconds (only processes new/changed files)
+
+💡 **First-Time User Note**: The initial import processes your entire conversation history. This is a one-time operation. After that, the system only imports new conversations, making it much faster and using less memory.
+
 ## Install
 
 ### Quick Start (Local Mode - Default)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-self-reflect",
-  "version": "2.4.8",
+  "version": "2.4.10",
   "description": "Give Claude perfect memory of all your conversations - Installation wizard for Python MCP server",
   "keywords": [
     "claude",

package/scripts/import-conversations-unified.py

@@ -8,6 +8,7 @@ import sys
 import json
 import glob
 import hashlib
+import gc
 from datetime import datetime
 from typing import List, Dict, Any
 import logging
@@ -29,7 +30,7 @@ from tenacity import (
 QDRANT_URL = os.getenv("QDRANT_URL", "http://localhost:6333")
 LOGS_DIR = os.getenv("LOGS_DIR", "/logs")
 STATE_FILE = os.getenv("STATE_FILE", "/config/imported-files.json")
-BATCH_SIZE = int(os.getenv("BATCH_SIZE", "100"))
+BATCH_SIZE = int(os.getenv("BATCH_SIZE", "10"))  # Reduced from 100 to prevent OOM
 PREFER_LOCAL_EMBEDDINGS = os.getenv("PREFER_LOCAL_EMBEDDINGS", "false").lower() == "true"
 VOYAGE_API_KEY = os.getenv("VOYAGE_KEY")
 
@@ -37,6 +38,60 @@ VOYAGE_API_KEY = os.getenv("VOYAGE_KEY")
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 logger = logging.getLogger(__name__)
 
+# State management functions
+def load_state():
+    """Load the import state from file."""
+    if os.path.exists(STATE_FILE):
+        try:
+            with open(STATE_FILE, 'r') as f:
+                state = json.load(f)
+                # Ensure the expected structure exists
+                if "imported_files" not in state:
+                    state["imported_files"] = {}
+                return state
+        except Exception as e:
+            logger.warning(f"Failed to load state file: {e}")
+    return {"imported_files": {}}
+
+def save_state(state):
+    """Save the import state to file."""
+    try:
+        # Ensure directory exists
+        os.makedirs(os.path.dirname(STATE_FILE), exist_ok=True)
+        # Write atomically by using a temp file
+        temp_file = STATE_FILE + ".tmp"
+        with open(temp_file, 'w') as f:
+            json.dump(state, f, indent=2)
+        os.replace(temp_file, STATE_FILE)
+        logger.debug(f"Saved state with {len(state['imported_files'])} files")
+    except Exception as e:
+        logger.error(f"Failed to save state file: {e}")
+
+def should_import_file(file_path, state):
+    """Check if a file should be imported based on modification time."""
+    str_path = str(file_path)
+    file_mtime = os.path.getmtime(file_path)
+    
+    if str_path in state["imported_files"]:
+        last_imported = state["imported_files"][str_path].get("last_imported", 0)
+        last_modified = state["imported_files"][str_path].get("last_modified", 0)
+        
+        # Skip if file hasn't been modified since last import
+        if file_mtime <= last_modified and last_imported > 0:
+            logger.info(f"Skipping unchanged file: {file_path.name}")
+            return False
+    
+    return True
+
+def update_file_state(file_path, state, chunks_imported):
+    """Update the state for an imported file."""
+    str_path = str(file_path)
+    state["imported_files"][str_path] = {
+        "last_modified": os.path.getmtime(file_path),
+        "last_imported": datetime.now().timestamp(),
+        "chunks_imported": chunks_imported
+    }
+
 # Initialize embedding provider
 embedding_provider = None
 embedding_dimension = None
@@ -120,7 +175,7 @@ def chunk_conversation(messages: List[Dict[str, Any]], chunk_size: int = 10) ->
     
     return chunks
 
-def import_project(project_path: Path, collection_name: str) -> int:
+def import_project(project_path: Path, collection_name: str, state: dict) -> int:
     """Import all conversations from a project."""
     jsonl_files = list(project_path.glob("*.jsonl"))
     
@@ -143,6 +198,10 @@ def import_project(project_path: Path, collection_name: str) -> int:
     total_chunks = 0
     
     for jsonl_file in jsonl_files:
+        # Check if file should be imported
+        if not should_import_file(jsonl_file, state):
+            continue
+            
         logger.info(f"Processing file: {jsonl_file.name}")
         try:
             # Read JSONL file and extract messages
@@ -241,7 +300,17 @@ def import_project(project_path: Path, collection_name: str) -> int:
                 
                 total_chunks += len(points)
             
-            logger.info(f"Imported {len(chunks)} chunks from {jsonl_file.name}")
+            file_chunks = len(chunks)
+            logger.info(f"Imported {file_chunks} chunks from {jsonl_file.name}")
+            
+            # Update state for this file
+            update_file_state(jsonl_file, state, file_chunks)
+            
+            # Save state after each file to prevent loss on OOM
+            save_state(state)
+            
+            # Force garbage collection to free memory
+            gc.collect()
             
         except Exception as e:
             logger.error(f"Failed to import {jsonl_file}: {e}")
@@ -258,6 +327,10 @@ def main():
         logger.error(f"Logs directory not found: {LOGS_DIR}")
         return
     
+    # Load existing state
+    state = load_state()
+    logger.info(f"Loaded state with {len(state['imported_files'])} previously imported files")
+    
     # Find all project directories
     project_dirs = [d for d in logs_path.iterdir() if d.is_dir()]
     
@@ -274,9 +347,15 @@ def main():
         collection_name = f"conv_{hashlib.md5(project_dir.name.encode()).hexdigest()[:8]}{collection_suffix}"
         
         logger.info(f"Importing project: {project_dir.name} -> {collection_name}")
-        chunks = import_project(project_dir, collection_name)
+        chunks = import_project(project_dir, collection_name, state)
         total_imported += chunks
         logger.info(f"Imported {chunks} chunks from {project_dir.name}")
+        
+        # Save state after each project to avoid losing progress
+        save_state(state)
+    
+    # Final save (redundant but ensures state is saved)
+    save_state(state)
     
     logger.info(f"Import complete! Total chunks imported: {total_imported}")