claude-self-reflect 2.7.4 → 2.8.0

package/Dockerfile.safe-watcher

@@ -30,8 +30,11 @@ RUN mkdir -p /root/.cache/fastembed && \
 # Set working directory
 WORKDIR /app
 
-# Copy scripts
-COPY scripts/ /scripts/
+# Copy application scripts
+COPY scripts/ /app/scripts/
+
+# Make watcher-loop.sh executable
+RUN chmod +x /app/scripts/watcher-loop.sh
 
 # Create config directory
 RUN mkdir -p /config
@@ -41,4 +44,4 @@ ENV PYTHONUNBUFFERED=1
 ENV MALLOC_ARENA_MAX=2
 
 # Run the watcher loop
-CMD ["/scripts/watcher-loop.sh"]
+CMD ["/app/scripts/watcher-loop.sh"]

package/README.md

@@ -149,10 +149,17 @@ Here's how your conversations get imported and prioritized:
 
 ![Import Architecture](docs/diagrams/import-architecture.png)
 
-**The system intelligently prioritizes your conversations:**
-- **HOT** (< 5 minutes): Switches to 2-second intervals for near real-time import
-- **🌡️ WARM** (< 24 hours): Normal priority, processed every 60 seconds
-- **❄️ COLD** (> 24 hours): Batch processed, max 5 per cycle to prevent blocking
+**The system intelligently processes your conversations:**
+- Runs every 60 seconds checking for new conversations
+- Processes newest conversations first (delta import pattern)
+- Maintains low memory usage (<50MB) through streaming
+- Handles up to 5 files per cycle to prevent blocking
+
+**HOT/WARM/COLD Intelligent Prioritization:**
+- **🔥 HOT** (< 5 minutes): Switches to 2-second intervals for near real-time import
+- **🌡️ WARM** (< 24 hours): Normal priority with starvation prevention (urgent after 30 min wait)
+- **❄️ COLD** (> 24 hours): Batch processed, max 5 per cycle to prevent blocking new content
+- Files are categorized by age and processed with priority queuing to ensure newest content gets imported quickly while preventing older files from being starved
 
 ## Using It
 

package/docker-compose.yaml

@@ -177,21 +177,29 @@ services:
       - ./scripts:/scripts:ro
     environment:
       - QDRANT_URL=http://qdrant:6333
-      - STATE_FILE=/config/watcher-state.json
+      - STATE_FILE=/config/csr-watcher.json
+      - LOGS_DIR=/logs  # Fixed: Point to mounted volume
       - VOYAGE_KEY=${VOYAGE_KEY:-}
       - PREFER_LOCAL_EMBEDDINGS=${PREFER_LOCAL_EMBEDDINGS:-true}
-      - HOT_WINDOW_MINUTES=${HOT_WINDOW_MINUTES:-15}
-      - MAX_COLD_FILES_PER_CYCLE=${MAX_COLD_FILES_PER_CYCLE:-3}
-      - MAX_MEMORY_MB=${MAX_MEMORY_MB:-300}
-      - WATCH_INTERVAL_SECONDS=${WATCH_INTERVAL_SECONDS:-30}
-      - MAX_FILES_PER_CYCLE=${MAX_FILES_PER_CYCLE:-10}
+      - ENABLE_MEMORY_DECAY=${ENABLE_MEMORY_DECAY:-false}
+      - DECAY_WEIGHT=${DECAY_WEIGHT:-0.3}
+      - DECAY_SCALE_DAYS=${DECAY_SCALE_DAYS:-90}
+      - CHECK_INTERVAL_S=${CHECK_INTERVAL_S:-60}
+      - HOT_CHECK_INTERVAL_S=${HOT_CHECK_INTERVAL_S:-2}
+      - HOT_WINDOW_MINUTES=${HOT_WINDOW_MINUTES:-5}
+      - WARM_WINDOW_HOURS=${WARM_WINDOW_HOURS:-24}
+      - MAX_COLD_FILES=${MAX_COLD_FILES:-5}
+      - MAX_WARM_WAIT_MINUTES=${MAX_WARM_WAIT_MINUTES:-30}
+      - MAX_MESSAGES_PER_CHUNK=${MAX_MESSAGES_PER_CHUNK:-10}
       - MAX_CHUNK_SIZE=${MAX_CHUNK_SIZE:-50}  # Messages per chunk for streaming
+      - MEMORY_LIMIT_MB=${MEMORY_LIMIT_MB:-1000}
+      - MEMORY_WARNING_MB=${MEMORY_WARNING_MB:-500}
       - PYTHONUNBUFFERED=1
       - MALLOC_ARENA_MAX=2
-    restart: "no"  # Manual start only - prevent system overload
-    profiles: ["safe-watch"]  # Requires explicit profile to run
-    mem_limit: 600m  # Increased from 400m to handle large files safely
-    memswap_limit: 600m
+    restart: unless-stopped
+    profiles: ["safe-watch", "watch"]  # Requires explicit profile to run
+    mem_limit: 1g  # Increased to 1GB to match MEMORY_LIMIT_MB
+    memswap_limit: 1g
     cpus: 1.0  # Single CPU core limit
 
   # MCP server for Claude integration

package/installer/setup-wizard-docker.js

@@ -454,6 +454,26 @@ async function enrichMetadata() {
   }
 }
 
+async function startWatcher() {
+  console.log('\n🔄 Starting the streaming watcher...');
+  console.log('   • HOT files (<5 min): 2-second processing');
+  console.log('   • WARM files (<24 hrs): Normal priority');
+  console.log('   • COLD files (>24 hrs): Batch processing');
+  
+  try {
+    safeExec('docker', ['compose', '--profile', 'watch', 'up', '-d', 'safe-watcher'], {
+      cwd: projectRoot,
+      stdio: 'inherit'
+    });
+    console.log('✅ Watcher started successfully!');
+    return true;
+  } catch (error) {
+    console.log('⚠️  Could not start watcher automatically');
+    console.log('   You can start it manually with: docker compose --profile watch up -d');
+    return false;
+  }
+}
+
 async function showFinalInstructions() {
   console.log('\n✅ Setup complete!');
   
@@ -461,7 +481,7 @@ async function showFinalInstructions() {
   console.log('   • 🌐 Qdrant Dashboard: http://localhost:6333/dashboard/');
   console.log('   • 📊 Status: All services running');
   console.log('   • 🔍 Search: Semantic search with memory decay enabled');
-  console.log('   • 🚀 Import: Watcher checking every 60 seconds');
+  console.log('   • 🚀 Watcher: HOT/WARM/COLD prioritization active');
   
   console.log('\n📋 Quick Reference Commands:');
   console.log('   • Check status: docker compose ps');
@@ -568,6 +588,9 @@ async function main() {
   // Enrich metadata (new in v2.5.19)
   await enrichMetadata();
   
+  // Start the watcher
+  await startWatcher();
+  
   // Show final instructions
   await showFinalInstructions();
   

package/mcp-server/src/server.py

@@ -9,6 +9,7 @@ import json
 import numpy as np
 import hashlib
 import time
+import logging
 
 from fastmcp import FastMCP, Context
 from .utils import normalize_project_name
@@ -124,18 +125,48 @@ indexing_status = {
     "is_checking": False
 }
 
-async def update_indexing_status():
+# Cache for indexing status (5-second TTL)
+_indexing_cache = {"result": None, "timestamp": 0}
+
+# Setup logger
+logger = logging.getLogger(__name__)
+
+def normalize_path(path_str: str) -> str:
+    """Normalize path for consistent comparison across platforms.
+    
+    Args:
+        path_str: Path string to normalize
+        
+    Returns:
+        Normalized path string with consistent separators
+    """
+    if not path_str:
+        return path_str
+    p = Path(path_str).expanduser().resolve()
+    return str(p).replace('\\', '/')  # Consistent separators for all platforms
+
+async def update_indexing_status(cache_ttl: int = 5):
     """Update indexing status by checking JSONL files vs Qdrant collections.
-    This is a lightweight check that compares file counts, not full content."""
-    global indexing_status
+    This is a lightweight check that compares file counts, not full content.
+    
+    Args:
+        cache_ttl: Cache time-to-live in seconds (default: 5)
+    """
+    global indexing_status, _indexing_cache
+    
+    # Check cache first (5-second TTL to prevent performance issues)
+    current_time = time.time()
+    if _indexing_cache["result"] and current_time - _indexing_cache["timestamp"] < cache_ttl:
+        # Use cached result
+        indexing_status = _indexing_cache["result"].copy()
+        return
     
     # Don't run concurrent checks
     if indexing_status["is_checking"]:
         return
     
-    # Only check every 5 minutes to avoid overhead
-    current_time = time.time()
-    if current_time - indexing_status["last_check"] < 300:  # 5 minutes
+    # Check immediately on first call, then every 60 seconds to avoid overhead
+    if indexing_status["last_check"] > 0 and current_time - indexing_status["last_check"] < 60:  # 1 minute
         return
     
     indexing_status["is_checking"] = True
@@ -151,46 +182,107 @@ async def update_indexing_status():
             jsonl_files = list(projects_dir.glob("**/*.jsonl"))
             total_files = len(jsonl_files)
             
-            # Check imported-files.json to see what's been imported
-            # The streaming importer uses imported-files.json with nested structure
-            # Try multiple possible locations for the config file
+            # Check imported-files.json AND watcher state files to see what's been imported
+            # The system uses multiple state files that need to be merged
+            all_imported_files = set()  # Use set to avoid duplicates
+            file_metadata = {}
+            
+            # 1. Check imported-files.json (batch importer)
             possible_paths = [
                 Path.home() / ".claude-self-reflect" / "config" / "imported-files.json",
                 Path(__file__).parent.parent.parent / "config" / "imported-files.json",
                 Path("/config/imported-files.json")  # Docker path if running in container
             ]
             
-            imported_files_path = None
             for path in possible_paths:
                 if path.exists():
-                    imported_files_path = path
-                    break
+                    try:
+                        with open(path, 'r') as f:
+                            imported_data = json.load(f)
+                            imported_files_dict = imported_data.get("imported_files", {})
+                            file_metadata.update(imported_data.get("file_metadata", {}))
+                            # Normalize paths before adding to set
+                            normalized_files = {normalize_path(k) for k in imported_files_dict.keys()}
+                            all_imported_files.update(normalized_files)
+                    except (json.JSONDecodeError, IOError) as e:
+                        logger.debug(f"Failed to read state file {path}: {e}")
+                        pass  # Continue if file is corrupted
             
-            if imported_files_path and imported_files_path.exists():
-                with open(imported_files_path, 'r') as f:
-                    imported_data = json.load(f)
-                    # The actual structure has imported_files and file_metadata at the top level
-                    # NOT nested under stream_position as previously assumed
-                    imported_files_dict = imported_data.get("imported_files", {})
-                    file_metadata = imported_data.get("file_metadata", {})
-                    
-                    # Convert dict keys to list for compatibility with existing logic
-                    imported_files_list = list(imported_files_dict.keys())
+            # 2. Check csr-watcher.json (streaming watcher - local mode)
+            watcher_paths = [
+                Path.home() / ".claude-self-reflect" / "config" / "csr-watcher.json",
+                Path("/config/csr-watcher.json")  # Docker path
+            ]
+            
+            for path in watcher_paths:
+                if path.exists():
+                    try:
+                        with open(path, 'r') as f:
+                            watcher_data = json.load(f)
+                            watcher_files = watcher_data.get("imported_files", {})
+                            # Normalize paths before adding to set
+                            normalized_files = {normalize_path(k) for k in watcher_files.keys()}
+                            all_imported_files.update(normalized_files)
+                            # Add to metadata with normalized paths
+                            for file_path, info in watcher_files.items():
+                                normalized = normalize_path(file_path)
+                                if normalized not in file_metadata:
+                                    file_metadata[normalized] = {
+                                        "position": 1,
+                                        "chunks": info.get("chunks", 0)
+                                    }
+                    except (json.JSONDecodeError, IOError) as e:
+                        logger.debug(f"Failed to read watcher state file {path}: {e}")
+                        pass  # Continue if file is corrupted
+            
+            # 3. Check csr-watcher-cloud.json (streaming watcher - cloud mode)
+            cloud_watcher_path = Path.home() / ".claude-self-reflect" / "config" / "csr-watcher-cloud.json"
+            if cloud_watcher_path.exists():
+                try:
+                    with open(cloud_watcher_path, 'r') as f:
+                        cloud_data = json.load(f)
+                        cloud_files = cloud_data.get("imported_files", {})
+                        # Normalize paths before adding to set
+                        normalized_files = {normalize_path(k) for k in cloud_files.keys()}
+                        all_imported_files.update(normalized_files)
+                        # Add to metadata with normalized paths
+                        for file_path, info in cloud_files.items():
+                            normalized = normalize_path(file_path)
+                            if normalized not in file_metadata:
+                                file_metadata[normalized] = {
+                                    "position": 1,
+                                    "chunks": info.get("chunks", 0)
+                                }
+                except (json.JSONDecodeError, IOError) as e:
+                    logger.debug(f"Failed to read cloud watcher state file {cloud_watcher_path}: {e}")
+                    pass  # Continue if file is corrupted
+            
+            # Convert set to list for compatibility
+            imported_files_list = list(all_imported_files)
                     
                     # Count files that have been imported
                     for file_path in jsonl_files:
+                        # Normalize the current file path for consistent comparison
+                        normalized_file = normalize_path(str(file_path))
+                        
                         # Try multiple path formats to match Docker's state file
                         file_str = str(file_path).replace(str(Path.home()), "/logs").replace("\\", "/")
                         # Also try without .claude/projects prefix (Docker mounts directly)
                         file_str_alt = file_str.replace("/.claude/projects", "")
                         
+                        # Normalize alternative paths as well
+                        normalized_alt = normalize_path(file_str)
+                        normalized_alt2 = normalize_path(file_str_alt)
+                        
                         # Check if file is in imported_files list (fully imported)
-                        if file_str in imported_files_list or file_str_alt in imported_files_list:
+                        if normalized_file in imported_files_list or normalized_alt in imported_files_list or normalized_alt2 in imported_files_list:
                             indexed_files += 1
                         # Or if it has metadata with position > 0 (partially imported)
-                        elif file_str in file_metadata and file_metadata[file_str].get("position", 0) > 0:
+                        elif normalized_file in file_metadata and file_metadata[normalized_file].get("position", 0) > 0:
+                            indexed_files += 1
+                        elif normalized_alt in file_metadata and file_metadata[normalized_alt].get("position", 0) > 0:
                             indexed_files += 1
-                        elif file_str_alt in file_metadata and file_metadata[file_str_alt].get("position", 0) > 0:
+                        elif normalized_alt2 in file_metadata and file_metadata[normalized_alt2].get("position", 0) > 0:
                             indexed_files += 1
         
         # Update status
@@ -203,9 +295,14 @@ async def update_indexing_status():
             indexing_status["percentage"] = (indexed_files / total_files) * 100
         else:
             indexing_status["percentage"] = 100.0
+        
+        # Update cache
+        _indexing_cache["result"] = indexing_status.copy()
+        _indexing_cache["timestamp"] = current_time
             
     except Exception as e:
         print(f"[WARNING] Failed to update indexing status: {e}")
+        logger.error(f"Failed to update indexing status: {e}", exc_info=True)
     finally:
         indexing_status["is_checking"] = False
     

package/mcp-server/src/status.py

@@ -5,6 +5,7 @@ Designed for <20ms execution time to support status bars and shell scripts.
 """
 
 import json
+import time
 from pathlib import Path
 from collections import defaultdict
 
@@ -53,11 +54,36 @@ def normalize_file_path(file_path: str) -> str:
     return file_path
 
 
+def get_watcher_status() -> dict:
+    """Get streaming watcher status if available."""
+    watcher_state_file = Path.home() / "config" / "csr-watcher.json"
+    
+    if not watcher_state_file.exists():
+        return {"running": False, "status": "not configured"}
+    
+    try:
+        with open(watcher_state_file) as f:
+            state = json.load(f)
+            
+        # Check if watcher is active (modified recently)
+        file_age = time.time() - watcher_state_file.stat().st_mtime
+        is_active = file_age < 120  # Active if updated in last 2 minutes
+        
+        return {
+            "running": is_active,
+            "files_processed": len(state.get("imported_files", {})),
+            "last_update_seconds": int(file_age),
+            "status": "🟢 active" if is_active else "🔴 inactive"
+        }
+    except:
+        return {"running": False, "status": "error reading state"}
+
+
 def get_status() -> dict:
     """Get indexing status with overall stats and per-project breakdown.
     
     Returns:
-        dict: JSON structure with overall and per-project indexing status
+        dict: JSON structure with overall and per-project indexing status, plus watcher status
     """
     projects_dir = Path.home() / ".claude" / "projects"
     project_stats = defaultdict(lambda: {"indexed": 0, "total": 0})
@@ -154,6 +180,9 @@ def get_status() -> dict:
             "total": stats["total"]
         }
     
+    # Add watcher status
+    result["watcher"] = get_watcher_status()
+    
     return result
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-self-reflect",
-  "version": "2.7.4",
+  "version": "2.8.0",
   "description": "Give Claude perfect memory of all your conversations - Installation wizard for Python MCP server",
   "keywords": [
     "claude",

package/scripts/import-conversations-unified.backup.py

@@ -1,374 +0,0 @@
-#!/usr/bin/env python3
-"""
-Streaming importer with true line-by-line processing to prevent OOM.
-Processes JSONL files without loading entire file into memory.
-"""
-
-import json
-import os
-import sys
-import hashlib
-import gc
-from pathlib import Path
-from datetime import datetime
-from typing import List, Dict, Any, Optional
-import logging
-
-# Add the project root to the Python path
-project_root = Path(__file__).parent.parent
-sys.path.insert(0, str(project_root))
-
-from qdrant_client import QdrantClient
-from qdrant_client.models import PointStruct, Distance, VectorParams
-
-# Set up logging
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s - %(levelname)s - %(message)s'
-)
-logger = logging.getLogger(__name__)
-
-# Environment variables
-QDRANT_URL = os.getenv("QDRANT_URL", "http://localhost:6333")
-STATE_FILE = os.getenv("STATE_FILE", "/config/imported-files.json")
-PREFER_LOCAL_EMBEDDINGS = os.getenv("PREFER_LOCAL_EMBEDDINGS", "true").lower() == "true"
-VOYAGE_API_KEY = os.getenv("VOYAGE_KEY")
-MAX_CHUNK_SIZE = int(os.getenv("MAX_CHUNK_SIZE", "50"))  # Messages per chunk
-
-# Initialize Qdrant client
-client = QdrantClient(url=QDRANT_URL)
-
-# Initialize embedding provider
-embedding_provider = None
-embedding_dimension = None
-
-if PREFER_LOCAL_EMBEDDINGS or not VOYAGE_API_KEY:
-    logger.info("Using local embeddings (fastembed)")
-    from fastembed import TextEmbedding
-    embedding_provider = TextEmbedding(model_name="sentence-transformers/all-MiniLM-L6-v2")
-    embedding_dimension = 384
-    collection_suffix = "local"
-else:
-    logger.info("Using Voyage AI embeddings")
-    import voyageai
-    embedding_provider = voyageai.Client(api_key=VOYAGE_API_KEY)
-    embedding_dimension = 1024
-    collection_suffix = "voyage"
-
-def normalize_project_name(project_name: str) -> str:
-    """Normalize project name for consistency."""
-    return project_name.replace("-Users-ramakrishnanannaswamy-projects-", "").replace("-", "_").lower()
-
-def get_collection_name(project_path: Path) -> str:
-    """Generate collection name from project path."""
-    normalized = normalize_project_name(project_path.name)
-    name_hash = hashlib.md5(normalized.encode()).hexdigest()[:8]
-    return f"conv_{name_hash}_{collection_suffix}"
-
-def ensure_collection(collection_name: str):
-    """Ensure collection exists with correct configuration."""
-    collections = client.get_collections().collections
-    if not any(c.name == collection_name for c in collections):
-        logger.info(f"Creating collection: {collection_name}")
-        client.create_collection(
-            collection_name=collection_name,
-            vectors_config=VectorParams(size=embedding_dimension, distance=Distance.COSINE)
-        )
-
-def generate_embeddings(texts: List[str]) -> List[List[float]]:
-    """Generate embeddings for texts."""
-    if PREFER_LOCAL_EMBEDDINGS or not VOYAGE_API_KEY:
-        embeddings = list(embedding_provider.passage_embed(texts))
-        return [emb.tolist() if hasattr(emb, 'tolist') else emb for emb in embeddings]
-    else:
-        response = embedding_provider.embed(texts, model="voyage-3")
-        return response.embeddings
-
-def process_and_upload_chunk(messages: List[Dict[str, Any]], chunk_index: int,
-                            conversation_id: str, created_at: str,
-                            metadata: Dict[str, Any], collection_name: str,
-                            project_path: Path) -> int:
-    """Process and immediately upload a single chunk."""
-    if not messages:
-        return 0
-    
-    # Extract text content
-    texts = []
-    for msg in messages:
-        role = msg.get("role", "unknown")
-        content = msg.get("content", "")
-        if content:
-            texts.append(f"{role.upper()}: {content}")
-    
-    if not texts:
-        return 0
-    
-    chunk_text = "\n".join(texts)
-    
-    try:
-        # Generate embedding
-        embeddings = generate_embeddings([chunk_text])
-        
-        # Create point ID
-        point_id = hashlib.md5(
-            f"{conversation_id}_{chunk_index}".encode()
-        ).hexdigest()[:16]
-        
-        # Create payload
-        payload = {
-            "text": chunk_text,
-            "conversation_id": conversation_id,
-            "chunk_index": chunk_index,
-            "timestamp": created_at,
-            "project": normalize_project_name(project_path.name),
-            "start_role": messages[0].get("role", "unknown") if messages else "unknown",
-            "message_count": len(messages)
-        }
-        
-        # Add metadata
-        if metadata:
-            payload.update(metadata)
-        
-        # Create point
-        point = PointStruct(
-            id=int(point_id, 16) % (2**63),
-            vector=embeddings[0],
-            payload=payload
-        )
-        
-        # Upload immediately
-        client.upsert(
-            collection_name=collection_name,
-            points=[point],
-            wait=True
-        )
-        
-        return 1
-        
-    except Exception as e:
-        logger.error(f"Error processing chunk {chunk_index}: {e}")
-        return 0
-
-def extract_metadata_single_pass(file_path: str) -> tuple[Dict[str, Any], str]:
-    """Extract metadata in a single pass, return metadata and first timestamp."""
-    metadata = {
-        "files_analyzed": [],
-        "files_edited": [],
-        "tools_used": [],
-        "concepts": []
-    }
-    
-    first_timestamp = None
-    
-    try:
-        with open(file_path, 'r', encoding='utf-8') as f:
-            for line in f:
-                if not line.strip():
-                    continue
-                    
-                try:
-                    data = json.loads(line)
-                    
-                    # Get timestamp from first valid entry
-                    if first_timestamp is None and 'timestamp' in data:
-                        first_timestamp = data.get('timestamp')
-                    
-                    # Extract tool usage from messages
-                    if 'message' in data and data['message']:
-                        msg = data['message']
-                        if msg.get('content'):
-                            content = msg['content']
-                            if isinstance(content, list):
-                                for item in content:
-                                    if isinstance(item, dict) and item.get('type') == 'tool_use':
-                                        tool_name = item.get('name', '')
-                                        if tool_name and tool_name not in metadata['tools_used']:
-                                            metadata['tools_used'].append(tool_name)
-                                        
-                                        # Extract file references
-                                        if 'input' in item:
-                                            input_data = item['input']
-                                            if isinstance(input_data, dict):
-                                                if 'file_path' in input_data:
-                                                    file_ref = input_data['file_path']
-                                                    if file_ref not in metadata['files_analyzed']:
-                                                        metadata['files_analyzed'].append(file_ref)
-                                                if 'path' in input_data:
-                                                    file_ref = input_data['path']
-                                                    if file_ref not in metadata['files_analyzed']:
-                                                        metadata['files_analyzed'].append(file_ref)
-                                        
-                except json.JSONDecodeError:
-                    continue
-                except Exception:
-                    continue
-                    
-    except Exception as e:
-        logger.warning(f"Error extracting metadata: {e}")
-    
-    return metadata, first_timestamp or datetime.now().isoformat()
-
-def stream_import_file(jsonl_file: Path, collection_name: str, project_path: Path) -> int:
-    """Stream import a single JSONL file without loading it into memory."""
-    logger.info(f"Streaming import of {jsonl_file.name}")
-    
-    # Extract metadata in first pass (lightweight)
-    metadata, created_at = extract_metadata_single_pass(str(jsonl_file))
-    
-    # Stream messages and process in chunks
-    chunk_buffer = []
-    chunk_index = 0
-    total_chunks = 0
-    conversation_id = jsonl_file.stem
-    
-    try:
-        with open(jsonl_file, 'r', encoding='utf-8') as f:
-            for line_num, line in enumerate(f, 1):
-                line = line.strip()
-                if not line:
-                    continue
-                
-                try:
-                    data = json.loads(line)
-                    
-                    # Skip non-message lines
-                    if data.get('type') == 'summary':
-                        continue
-                    
-                    # Extract message if present
-                    if 'message' in data and data['message']:
-                        msg = data['message']
-                        if msg.get('role') and msg.get('content'):
-                            # Extract content
-                            content = msg['content']
-                            if isinstance(content, list):
-                                text_parts = []
-                                for item in content:
-                                    if isinstance(item, dict) and item.get('type') == 'text':
-                                        text_parts.append(item.get('text', ''))
-                                    elif isinstance(item, str):
-                                        text_parts.append(item)
-                                content = '\n'.join(text_parts)
-                            
-                            if content:
-                                chunk_buffer.append({
-                                    'role': msg['role'],
-                                    'content': content
-                                })
-                                
-                                # Process chunk when buffer reaches MAX_CHUNK_SIZE
-                                if len(chunk_buffer) >= MAX_CHUNK_SIZE:
-                                    chunks = process_and_upload_chunk(
-                                        chunk_buffer, chunk_index, conversation_id,
-                                        created_at, metadata, collection_name, project_path
-                                    )
-                                    total_chunks += chunks
-                                    chunk_buffer = []
-                                    chunk_index += 1
-                                    
-                                    # Force garbage collection after each chunk
-                                    gc.collect()
-                                    
-                                    # Log progress
-                                    if chunk_index % 10 == 0:
-                                        logger.info(f"Processed {chunk_index} chunks from {jsonl_file.name}")
-                                    
-                except json.JSONDecodeError:
-                    logger.debug(f"Skipping invalid JSON at line {line_num}")
-                except Exception as e:
-                    logger.debug(f"Error processing line {line_num}: {e}")
-        
-        # Process remaining messages
-        if chunk_buffer:
-            chunks = process_and_upload_chunk(
-                chunk_buffer, chunk_index, conversation_id,
-                created_at, metadata, collection_name, project_path
-            )
-            total_chunks += chunks
-        
-        logger.info(f"Imported {total_chunks} chunks from {jsonl_file.name}")
-        return total_chunks
-        
-    except Exception as e:
-        logger.error(f"Failed to import {jsonl_file}: {e}")
-        return 0
-
-def load_state() -> dict:
-    """Load import state."""
-    if os.path.exists(STATE_FILE):
-        try:
-            with open(STATE_FILE, 'r') as f:
-                return json.load(f)
-        except:
-            pass
-    return {"imported_files": {}}
-
-def save_state(state: dict):
-    """Save import state."""
-    os.makedirs(os.path.dirname(STATE_FILE), exist_ok=True)
-    with open(STATE_FILE, 'w') as f:
-        json.dump(state, f, indent=2)
-
-def should_import_file(file_path: Path, state: dict) -> bool:
-    """Check if file should be imported."""
-    file_str = str(file_path)
-    if file_str in state.get("imported_files", {}):
-        file_info = state["imported_files"][file_str]
-        last_modified = file_path.stat().st_mtime
-        if file_info.get("last_modified") == last_modified:
-            logger.info(f"Skipping unchanged file: {file_path.name}")
-            return False
-    return True
-
-def update_file_state(file_path: Path, state: dict, chunks: int):
-    """Update state for imported file."""
-    file_str = str(file_path)
-    state["imported_files"][file_str] = {
-        "imported_at": datetime.now().isoformat(),
-        "last_modified": file_path.stat().st_mtime,
-        "chunks": chunks
-    }
-
-def main():
-    """Main import function."""
-    # Load state
-    state = load_state()
-    logger.info(f"Loaded state with {len(state.get('imported_files', {}))} previously imported files")
-    
-    # Find all projects
-    logs_dir = Path(os.getenv("LOGS_DIR", "/logs"))
-    project_dirs = [d for d in logs_dir.iterdir() if d.is_dir()]
-    logger.info(f"Found {len(project_dirs)} projects to import")
-    
-    total_imported = 0
-    
-    for project_dir in project_dirs:
-        # Get collection name
-        collection_name = get_collection_name(project_dir)
-        logger.info(f"Importing project: {project_dir.name} -> {collection_name}")
-        
-        # Ensure collection exists
-        ensure_collection(collection_name)
-        
-        # Find JSONL files
-        jsonl_files = sorted(project_dir.glob("*.jsonl"))
-        
-        # Limit files per cycle if specified
-        max_files = int(os.getenv("MAX_FILES_PER_CYCLE", "1000"))
-        jsonl_files = jsonl_files[:max_files]
-        
-        for jsonl_file in jsonl_files:
-            if should_import_file(jsonl_file, state):
-                chunks = stream_import_file(jsonl_file, collection_name, project_dir)
-                if chunks > 0:
-                    update_file_state(jsonl_file, state, chunks)
-                    save_state(state)
-                    total_imported += 1
-                    
-                    # Force GC after each file
-                    gc.collect()
-    
-    logger.info(f"Import complete: processed {total_imported} files")
-
-if __name__ == "__main__":
-    main()

package/scripts/import-latest.py

@@ -1,124 +0,0 @@
-#!/usr/bin/env python3
-"""
-Quick import script for current project's latest conversations.
-Designed for PreCompact hook integration - targets <10 second imports.
-"""
-
-import os
-import sys
-import json
-import subprocess
-from datetime import datetime, timedelta
-from pathlib import Path
-import logging
-
-# Configuration
-LOGS_DIR = os.getenv("LOGS_DIR", os.path.expanduser("~/.claude/projects"))
-STATE_FILE = os.getenv("STATE_FILE", os.path.expanduser("~/.claude-self-reflect-state.json"))
-HOURS_BACK = int(os.getenv("IMPORT_HOURS_BACK", "2"))  # Only import last 2 hours by default
-
-# Set up logging
-logging.basicConfig(
-    level=logging.INFO, 
-    format='%(asctime)s - %(levelname)s - %(message)s'
-)
-logger = logging.getLogger(__name__)
-
-def load_state():
-    """Load import state from file."""
-    if os.path.exists(STATE_FILE):
-        try:
-            with open(STATE_FILE, 'r') as f:
-                return json.load(f)
-        except:
-            return {}
-    return {}
-
-def save_state(state):
-    """Save import state to file."""
-    os.makedirs(os.path.dirname(STATE_FILE), exist_ok=True)
-    with open(STATE_FILE, 'w') as f:
-        json.dump(state, f, indent=2)
-
-def get_project_from_cwd():
-    """Detect project from current working directory."""
-    cwd = os.getcwd()
-    # Convert path to project name format used in logs
-    # Claude logs use format: -Users-username-path-to-project
-    project_name = cwd.replace('/', '-')
-    # Keep the leading dash as that's how Claude stores it
-    if not project_name.startswith('-'):
-        project_name = '-' + project_name
-    return project_name
-
-def get_recent_files(project_path: Path, hours_back: int):
-    """Get JSONL files modified in the last N hours."""
-    cutoff_time = datetime.now() - timedelta(hours=hours_back)
-    recent_files = []
-    
-    for jsonl_file in project_path.glob("*.jsonl"):
-        mtime = datetime.fromtimestamp(jsonl_file.stat().st_mtime)
-        if mtime > cutoff_time:
-            recent_files.append(jsonl_file)
-    
-    return sorted(recent_files, key=lambda f: f.stat().st_mtime, reverse=True)
-
-def main():
-    """Main quick import function."""
-    start_time = datetime.now()
-    
-    # Detect current project
-    project_name = get_project_from_cwd()
-    project_path = Path(LOGS_DIR) / project_name
-    
-    if not project_path.exists():
-        logger.warning(f"Project logs not found: {project_path}")
-        logger.info("Make sure you're in a project directory with Claude conversations.")
-        return
-    
-    logger.info(f"Quick importing latest conversations for: {project_name}")
-    
-    # Get recent files
-    recent_files = get_recent_files(project_path, HOURS_BACK)
-    logger.info(f"Found {len(recent_files)} files modified in last {HOURS_BACK} hours")
-    
-    if not recent_files:
-        logger.info("No recent conversations to import")
-        return
-    
-    # For now, just call the unified importer with the specific project
-    # This is a temporary solution until we implement incremental imports
-    script_dir = os.path.dirname(os.path.abspath(__file__))
-    unified_script = os.path.join(script_dir, "import-conversations-unified.py")
-    
-    # Set environment to only process this project
-    env = os.environ.copy()
-    env['LOGS_DIR'] = str(project_path.parent)
-    env['IMPORT_PROJECT'] = project_name
-    
-    try:
-        # Run the unified importer for just this project
-        result = subprocess.run(
-            [sys.executable, unified_script],
-            env=env,
-            capture_output=True,
-            text=True,
-            timeout=60  # 60 second timeout
-        )
-        
-        if result.returncode == 0:
-            logger.info("Quick import completed successfully")
-        else:
-            logger.error(f"Import failed: {result.stderr}")
-            
-    except subprocess.TimeoutExpired:
-        logger.warning("Import timed out after 60 seconds")
-    except Exception as e:
-        logger.error(f"Error during import: {e}")
-    
-    # Report timing
-    elapsed = (datetime.now() - start_time).total_seconds()
-    logger.info(f"Quick import completed in {elapsed:.1f} seconds")
-
-if __name__ == "__main__":
-    main()

package/scripts/import-old-format.py

@@ -1,171 +0,0 @@
-#!/usr/bin/env python3
-"""
-Import old format JSONL files from Claude conversations.
-These files have a different structure with type/summary fields instead of messages.
-"""
-
-import json
-import sys
-from pathlib import Path
-import hashlib
-import uuid
-from datetime import datetime
-from qdrant_client import QdrantClient
-from qdrant_client.models import Distance, VectorParams, PointStruct
-from fastembed import TextEmbedding
-import logging
-
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
-logger = logging.getLogger(__name__)
-
-def import_old_format_project(project_dir: Path, project_path: str = None):
-    """Import old format JSONL files from a project directory."""
-    
-    # Initialize
-    client = QdrantClient(url='http://localhost:6333')
-    model = TextEmbedding(model_name='sentence-transformers/all-MiniLM-L6-v2', max_length=512)
-    
-    # Determine project path from directory name if not provided
-    if not project_path:
-        # Convert -Users-username-projects-projectname back to path
-        dir_name = project_dir.name
-        project_path = '/' + dir_name.strip('-').replace('-', '/')
-    
-    # Create collection name
-    project_hash = hashlib.md5(project_path.encode()).hexdigest()[:8]
-    collection_name = f'conv_{project_hash}_local'
-    
-    logger.info(f'Project: {project_path}')
-    logger.info(f'Collection: {collection_name}')
-    
-    # Create collection if needed
-    try:
-        client.get_collection(collection_name)
-        logger.info('Collection exists')
-    except:
-        client.create_collection(
-            collection_name=collection_name,
-            vectors_config=VectorParams(size=384, distance=Distance.COSINE)
-        )
-        logger.info('Created collection')
-    
-    # Process all JSONL files
-    jsonl_files = list(project_dir.glob('*.jsonl'))
-    logger.info(f'Found {len(jsonl_files)} files to import')
-    
-    total_points = 0
-    for file_path in jsonl_files:
-        logger.info(f'Processing {file_path.name}...')
-        points_batch = []
-        
-        with open(file_path, 'r', encoding='utf-8') as f:
-            conversation_text = []
-            file_timestamp = file_path.stat().st_mtime
-            
-            for line_num, line in enumerate(f, 1):
-                try:
-                    data = json.loads(line)
-                    msg_type = data.get('type', '')
-                    
-                    # Extract text content based on type
-                    content = None
-                    if msg_type == 'summary' and data.get('summary'):
-                        content = f"[Conversation Summary] {data['summary']}"
-                    elif msg_type == 'user' and data.get('summary'):
-                        content = f"User: {data['summary']}"
-                    elif msg_type == 'assistant' and data.get('summary'):
-                        content = f"Assistant: {data['summary']}"
-                    elif msg_type in ['user', 'assistant']:
-                        # Try to get content from other fields
-                        if 'content' in data:
-                            content = f"{msg_type.title()}: {data['content']}"
-                        elif 'text' in data:
-                            content = f"{msg_type.title()}: {data['text']}"
-                    
-                    if content:
-                        conversation_text.append(content)
-                        
-                        # Create chunks every 5 messages or at end
-                        if len(conversation_text) >= 5:
-                            chunk_text = '\n\n'.join(conversation_text)
-                            if chunk_text.strip():
-                                # Generate embedding
-                                embedding = list(model.embed([chunk_text[:2000]]))[0]  # Limit to 2000 chars
-                                
-                                point = PointStruct(
-                                    id=str(uuid.uuid4()),
-                                    vector=embedding.tolist(),
-                                    payload={
-                                        'content': chunk_text[:1000],  # Store first 1000 chars
-                                        'full_content': chunk_text[:4000],  # Store more for context
-                                        'project_path': project_path,
-                                        'file_path': str(file_path),
-                                        'file_name': file_path.name,
-                                        'conversation_id': file_path.stem,
-                                        'chunk_index': len(points_batch),
-                                        'timestamp': file_timestamp,
-                                        'type': 'conversation_chunk'
-                                    }
-                                )
-                                points_batch.append(point)
-                                conversation_text = []
-                
-                except json.JSONDecodeError:
-                    logger.warning(f'Invalid JSON at line {line_num} in {file_path.name}')
-                except Exception as e:
-                    logger.warning(f'Error processing line {line_num}: {e}')
-            
-            # Handle remaining text
-            if conversation_text:
-                chunk_text = '\n\n'.join(conversation_text)
-                if chunk_text.strip():
-                    embedding = list(model.embed([chunk_text[:2000]]))[0]
-                    
-                    point = PointStruct(
-                        id=str(uuid.uuid4()),
-                        vector=embedding.tolist(),
-                        payload={
-                            'content': chunk_text[:1000],
-                            'full_content': chunk_text[:4000],
-                            'project_path': project_path,
-                            'file_path': str(file_path),
-                            'file_name': file_path.name,
-                            'conversation_id': file_path.stem,
-                            'chunk_index': len(points_batch),
-                            'timestamp': file_timestamp,
-                            'type': 'conversation_chunk'
-                        }
-                    )
-                    points_batch.append(point)
-        
-        # Upload batch
-        if points_batch:
-            client.upsert(collection_name=collection_name, points=points_batch)
-            logger.info(f'  Uploaded {len(points_batch)} chunks from {file_path.name}')
-            total_points += len(points_batch)
-    
-    # Verify
-    info = client.get_collection(collection_name)
-    logger.info(f'\nImport complete!')
-    logger.info(f'Collection {collection_name} now has {info.points_count} points')
-    logger.info(f'Added {total_points} new points in this import')
-    
-    return collection_name, total_points
-
-def main():
-    if len(sys.argv) < 2:
-        print("Usage: python import-old-format.py <project-directory> [project-path]")
-        print("Example: python import-old-format.py ~/.claude/projects/-Users-me-projects-myapp /Users/me/projects/myapp")
-        sys.exit(1)
-    
-    project_dir = Path(sys.argv[1]).expanduser()
-    project_path = sys.argv[2] if len(sys.argv) > 2 else None
-    
-    if not project_dir.exists():
-        print(f"Error: Directory {project_dir} does not exist")
-        sys.exit(1)
-    
-    import_old_format_project(project_dir, project_path)
-
-if __name__ == "__main__":
-    main()