claude-self-reflect 2.3.2 → 2.3.4

package/mcp-server/src/server.py

@@ -1,6 +1,7 @@
 """Claude Reflect MCP Server with Memory Decay."""
 
 import os
+import asyncio
 from pathlib import Path
 from typing import Any, Optional, List, Dict, Union
 from datetime import datetime
@@ -30,8 +31,25 @@ DECAY_WEIGHT = float(os.getenv('DECAY_WEIGHT', '0.3'))
 DECAY_SCALE_DAYS = float(os.getenv('DECAY_SCALE_DAYS', '90'))
 USE_NATIVE_DECAY = os.getenv('USE_NATIVE_DECAY', 'false').lower() == 'true'
 
-# Initialize Voyage AI client
-voyage_client = voyageai.Client(api_key=VOYAGE_API_KEY) if VOYAGE_API_KEY else None
+# Embedding configuration
+PREFER_LOCAL_EMBEDDINGS = os.getenv('PREFER_LOCAL_EMBEDDINGS', 'false').lower() == 'true'
+EMBEDDING_MODEL = os.getenv('EMBEDDING_MODEL', 'sentence-transformers/all-MiniLM-L6-v2')
+
+# Initialize Voyage AI client (only if not using local embeddings)
+voyage_client = None
+if not PREFER_LOCAL_EMBEDDINGS and VOYAGE_API_KEY:
+    voyage_client = voyageai.Client(api_key=VOYAGE_API_KEY)
+
+# Initialize local embedding model if needed
+local_embedding_model = None
+if PREFER_LOCAL_EMBEDDINGS or not VOYAGE_API_KEY:
+    try:
+        from fastembed import TextEmbedding
+        local_embedding_model = TextEmbedding(model_name=EMBEDDING_MODEL)
+        print(f"[DEBUG] Initialized local embedding model: {EMBEDDING_MODEL}")
+    except ImportError:
+        print("[ERROR] FastEmbed not available. Install with: pip install fastembed")
+        raise
 
 # Debug environment loading
 print(f"[DEBUG] Environment variables loaded:")
@@ -39,6 +57,8 @@ print(f"[DEBUG] ENABLE_MEMORY_DECAY: {ENABLE_MEMORY_DECAY}")
 print(f"[DEBUG] USE_NATIVE_DECAY: {USE_NATIVE_DECAY}")
 print(f"[DEBUG] DECAY_WEIGHT: {DECAY_WEIGHT}")
 print(f"[DEBUG] DECAY_SCALE_DAYS: {DECAY_SCALE_DAYS}")
+print(f"[DEBUG] PREFER_LOCAL_EMBEDDINGS: {PREFER_LOCAL_EMBEDDINGS}")
+print(f"[DEBUG] EMBEDDING_MODEL: {EMBEDDING_MODEL}")
 print(f"[DEBUG] env_path: {env_path}")
 
 
@@ -63,22 +83,50 @@ mcp = FastMCP(
 # Create Qdrant client
 qdrant_client = AsyncQdrantClient(url=QDRANT_URL)
     
-async def get_voyage_collections() -> List[str]:
-    """Get all Voyage collections."""
+async def get_all_collections() -> List[str]:
+    """Get all collections (both Voyage and local)."""
     collections = await qdrant_client.get_collections()
-    return [c.name for c in collections.collections if c.name.endswith('_voyage')]
+    # Support both _voyage and _local collections, plus reflections
+    return [c.name for c in collections.collections 
+            if c.name.endswith('_voyage') or c.name.endswith('_local') or c.name.startswith('reflections')]
 
 async def generate_embedding(text: str) -> List[float]:
-    """Generate embedding using Voyage AI."""
-    if not voyage_client:
-        raise ValueError("Voyage AI API key not configured")
-    
-    result = voyage_client.embed(
-        texts=[text],
-        model="voyage-3-large",
-        input_type="query"
-    )
-    return result.embeddings[0]
+    """Generate embedding using configured provider."""
+    if PREFER_LOCAL_EMBEDDINGS or not voyage_client:
+        # Use local embeddings
+        if not local_embedding_model:
+            raise ValueError("Local embedding model not initialized")
+        
+        # Run in executor since fastembed is synchronous
+        loop = asyncio.get_event_loop()
+        embeddings = await loop.run_in_executor(
+            None, lambda: list(local_embedding_model.embed([text]))
+        )
+        return embeddings[0].tolist()
+    else:
+        # Use Voyage AI
+        result = voyage_client.embed(
+            texts=[text],
+            model="voyage-3-large",
+            input_type="query"
+        )
+        return result.embeddings[0]
+
+def get_embedding_dimension() -> int:
+    """Get the dimension of embeddings based on the provider."""
+    if PREFER_LOCAL_EMBEDDINGS or not voyage_client:
+        # all-MiniLM-L6-v2 produces 384-dimensional embeddings
+        return 384
+    else:
+        # voyage-3-large produces 1024-dimensional embeddings
+        return 1024
+
+def get_collection_suffix() -> str:
+    """Get the collection suffix based on embedding provider."""
+    if PREFER_LOCAL_EMBEDDINGS or not voyage_client:
+        return "_local"
+    else:
+        return "_voyage"
     
 # Register tools
 @mcp.tool()
@@ -115,17 +163,18 @@ async def reflect_on_past(
         # Generate embedding
         query_embedding = await generate_embedding(query)
         
-        # Get all Voyage collections
-        voyage_collections = await get_voyage_collections()
-        if not voyage_collections:
+        # Get all collections
+        all_collections = await get_all_collections()
+        if not all_collections:
             return "No conversation collections found. Please import conversations first."
         
-        await ctx.debug(f"Searching across {len(voyage_collections)} collections")
+        await ctx.debug(f"Searching across {len(all_collections)} collections")
+        await ctx.debug(f"Using {'local' if PREFER_LOCAL_EMBEDDINGS or not voyage_client else 'Voyage AI'} embeddings")
         
         all_results = []
         
         # Search each collection
-        for collection_name in voyage_collections:
+        for collection_name in all_collections:
             try:
                 if should_use_decay and USE_NATIVE_DECAY:
                     # Use native Qdrant decay
@@ -179,7 +228,7 @@ async def reflect_on_past(
                             timestamp=point.payload.get('timestamp', datetime.now().isoformat()),
                             role=point.payload.get('start_role', point.payload.get('role', 'unknown')),
                             excerpt=(point.payload.get('text', '')[:500] + '...'),
-                            project_name=point.payload.get('project', collection_name.replace('conv_', '').replace('_voyage', '')),
+                            project_name=point.payload.get('project', collection_name.replace('conv_', '').replace('_voyage', '').replace('_local', '')),
                             conversation_id=point.payload.get('conversation_id'),
                             collection_name=collection_name
                         ))
@@ -240,7 +289,7 @@ async def reflect_on_past(
                             timestamp=point.payload.get('timestamp', datetime.now().isoformat()),
                             role=point.payload.get('start_role', point.payload.get('role', 'unknown')),
                             excerpt=(point.payload.get('text', '')[:500] + '...'),
-                            project_name=point.payload.get('project', collection_name.replace('conv_', '').replace('_voyage', '')),
+                            project_name=point.payload.get('project', collection_name.replace('conv_', '').replace('_voyage', '').replace('_local', '')),
                             conversation_id=point.payload.get('conversation_id'),
                             collection_name=collection_name
                         ))
@@ -261,7 +310,7 @@ async def reflect_on_past(
                             timestamp=point.payload.get('timestamp', datetime.now().isoformat()),
                             role=point.payload.get('start_role', point.payload.get('role', 'unknown')),
                             excerpt=(point.payload.get('text', '')[:500] + '...'),
-                            project_name=point.payload.get('project', collection_name.replace('conv_', '').replace('_voyage', '')),
+                            project_name=point.payload.get('project', collection_name.replace('conv_', '').replace('_voyage', '').replace('_local', '')),
                             conversation_id=point.payload.get('conversation_id'),
                             collection_name=collection_name
                         ))
@@ -302,8 +351,46 @@ async def store_reflection(
     """Store an important insight or reflection for future reference."""
     
     try:
-        # TODO: Implement actual storage in a dedicated reflections collection
-        # For now, just acknowledge the storage
+        # Create reflections collection name
+        collection_name = f"reflections{get_collection_suffix()}"
+        
+        # Ensure collection exists
+        try:
+            collection_info = await qdrant_client.get_collection(collection_name)
+        except:
+            # Create collection if it doesn't exist
+            await qdrant_client.create_collection(
+                collection_name=collection_name,
+                vectors_config=VectorParams(
+                    size=get_embedding_dimension(),
+                    distance=Distance.COSINE
+                )
+            )
+            await ctx.debug(f"Created reflections collection: {collection_name}")
+        
+        # Generate embedding for the reflection
+        embedding = await generate_embedding(content)
+        
+        # Create point with metadata
+        point_id = datetime.now().timestamp()
+        point = PointStruct(
+            id=int(point_id),
+            vector=embedding,
+            payload={
+                "text": content,
+                "tags": tags,
+                "timestamp": datetime.now().isoformat(),
+                "type": "reflection",
+                "role": "user_reflection"
+            }
+        )
+        
+        # Store in Qdrant
+        await qdrant_client.upsert(
+            collection_name=collection_name,
+            points=[point]
+        )
+        
         tags_str = ', '.join(tags) if tags else 'none'
         return f"Reflection stored successfully with tags: {tags_str}"
         

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-self-reflect",
-  "version": "2.3.2",
+  "version": "2.3.4",
   "description": "Give Claude perfect memory of all your conversations - Installation wizard for Python MCP server",
   "keywords": [
     "claude",

package/scripts/import-conversations-unified.py

@@ -0,0 +1,269 @@
+#!/usr/bin/env python3
+"""
+Unified import script that supports both local and Voyage AI embeddings.
+"""
+
+import os
+import sys
+import json
+import glob
+import hashlib
+from datetime import datetime
+from typing import List, Dict, Any
+import logging
+from pathlib import Path
+
+from qdrant_client import QdrantClient
+from qdrant_client.models import (
+    VectorParams, Distance, PointStruct,
+    Filter, FieldCondition, MatchValue
+)
+
+# Configuration
+QDRANT_URL = os.getenv("QDRANT_URL", "http://localhost:6333")
+LOGS_DIR = os.getenv("LOGS_DIR", "/logs")
+STATE_FILE = os.getenv("STATE_FILE", "/config/imported-files.json")
+BATCH_SIZE = int(os.getenv("BATCH_SIZE", "100"))
+PREFER_LOCAL_EMBEDDINGS = os.getenv("PREFER_LOCAL_EMBEDDINGS", "false").lower() == "true"
+VOYAGE_API_KEY = os.getenv("VOYAGE_KEY")
+
+# Set up logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+# Initialize embedding provider
+embedding_provider = None
+embedding_dimension = None
+collection_suffix = None
+
+if PREFER_LOCAL_EMBEDDINGS or not VOYAGE_API_KEY:
+    # Use local embeddings
+    logger.info("Using local embeddings (fastembed)")
+    from fastembed import TextEmbedding
+    embedding_provider = TextEmbedding("sentence-transformers/all-MiniLM-L6-v2")
+    embedding_dimension = 384
+    collection_suffix = "_local"
+else:
+    # Use Voyage AI
+    logger.info("Using Voyage AI embeddings")
+    import voyageai
+    voyage_client = voyageai.Client(api_key=VOYAGE_API_KEY)
+    embedding_dimension = 1024
+    collection_suffix = "_voyage"
+
+# Initialize Qdrant client
+client = QdrantClient(url=QDRANT_URL)
+
+def generate_embeddings(texts: List[str]) -> List[List[float]]:
+    """Generate embeddings for a list of texts."""
+    if PREFER_LOCAL_EMBEDDINGS or not VOYAGE_API_KEY:
+        # Local embeddings using FastEmbed
+        embeddings = list(embedding_provider.passage_embed(texts))
+        return [embedding.tolist() for embedding in embeddings]
+    else:
+        # Voyage AI embeddings
+        result = voyage_client.embed(
+            texts=texts,
+            model="voyage-3-large",
+            input_type="document"
+        )
+        return result.embeddings
+
+def chunk_conversation(messages: List[Dict[str, Any]], chunk_size: int = 10) -> List[Dict[str, Any]]:
+    """Chunk conversation into smaller segments."""
+    chunks = []
+    
+    for i in range(0, len(messages), chunk_size):
+        chunk_messages = messages[i:i + chunk_size]
+        
+        # Extract text content
+        texts = []
+        for msg in chunk_messages:
+            role = msg.get("role", "unknown")
+            content = msg.get("content", "")
+            
+            if isinstance(content, list):
+                # Handle structured content
+                text_parts = []
+                for item in content:
+                    if isinstance(item, dict) and item.get("type") == "text":
+                        text_parts.append(item.get("text", ""))
+                    elif isinstance(item, str):
+                        text_parts.append(item)
+                content = " ".join(text_parts)
+            
+            if content:
+                texts.append(f"{role.upper()}: {content}")
+        
+        if texts:
+            chunks.append({
+                "text": "\n".join(texts),
+                "messages": chunk_messages,
+                "chunk_index": i // chunk_size,
+                "start_role": chunk_messages[0].get("role", "unknown") if chunk_messages else "unknown"
+            })
+    
+    return chunks
+
+def import_project(project_path: Path, collection_name: str) -> int:
+    """Import all conversations from a project."""
+    jsonl_files = list(project_path.glob("*.jsonl"))
+    
+    if not jsonl_files:
+        logger.warning(f"No JSONL files found in {project_path}")
+        return 0
+    
+    # Check if collection exists
+    collections = client.get_collections().collections
+    if collection_name not in [c.name for c in collections]:
+        logger.info(f"Creating collection: {collection_name}")
+        client.create_collection(
+            collection_name=collection_name,
+            vectors_config=VectorParams(
+                size=embedding_dimension,
+                distance=Distance.COSINE
+            )
+        )
+    
+    total_chunks = 0
+    
+    for jsonl_file in jsonl_files:
+        logger.info(f"Processing file: {jsonl_file.name}")
+        try:
+            # Read JSONL file and extract messages
+            messages = []
+            created_at = None
+            
+            with open(jsonl_file, 'r', encoding='utf-8') as f:
+                for line_num, line in enumerate(f, 1):
+                    line = line.strip()
+                    if not line:
+                        continue
+                    
+                    try:
+                        data = json.loads(line)
+                        
+                        # Extract timestamp from first message
+                        if created_at is None and 'timestamp' in data:
+                            created_at = data.get('timestamp')
+                        
+                        # Skip non-message lines (summaries, etc.)
+                        if data.get('type') == 'summary':
+                            continue
+                            
+                        # Extract message if present
+                        if 'message' in data and data['message']:
+                            msg = data['message']
+                            if msg.get('role') and msg.get('content'):
+                                # Handle content that's an array of objects
+                                content = msg['content']
+                                if isinstance(content, list):
+                                    text_parts = []
+                                    for item in content:
+                                        if isinstance(item, dict) and item.get('type') == 'text':
+                                            text_parts.append(item.get('text', ''))
+                                        elif isinstance(item, str):
+                                            text_parts.append(item)
+                                    content = '\n'.join(text_parts)
+                                
+                                if content:
+                                    messages.append({
+                                        'role': msg['role'],
+                                        'content': content
+                                    })
+                    except json.JSONDecodeError:
+                        logger.debug(f"Skipping invalid JSON at line {line_num}")
+                    except Exception as e:
+                        logger.error(f"Error processing line {line_num}: {e}")
+            
+            if not messages:
+                continue
+            
+            # Extract metadata
+            if created_at is None:
+                created_at = datetime.now().isoformat()
+            conversation_id = jsonl_file.stem
+            
+            # Chunk the conversation
+            chunks = chunk_conversation(messages)
+            
+            if not chunks:
+                continue
+            
+            # Process in batches
+            for batch_start in range(0, len(chunks), BATCH_SIZE):
+                batch = chunks[batch_start:batch_start + BATCH_SIZE]
+                texts = [chunk["text"] for chunk in batch]
+                
+                # Generate embeddings
+                embeddings = generate_embeddings(texts)
+                
+                # Create points
+                points = []
+                for chunk, embedding in zip(batch, embeddings):
+                    point_id = hashlib.md5(
+                        f"{conversation_id}_{chunk['chunk_index']}".encode()
+                    ).hexdigest()[:16]
+                    
+                    points.append(PointStruct(
+                        id=int(point_id, 16) % (2**63),  # Convert to valid integer ID
+                        vector=embedding,
+                        payload={
+                            "text": chunk["text"],
+                            "conversation_id": conversation_id,
+                            "chunk_index": chunk["chunk_index"],
+                            "timestamp": created_at,
+                            "project": project_path.name,
+                            "start_role": chunk["start_role"]
+                        }
+                    ))
+                
+                # Upload to Qdrant
+                client.upsert(
+                    collection_name=collection_name,
+                    points=points
+                )
+                
+                total_chunks += len(points)
+            
+            logger.info(f"Imported {len(chunks)} chunks from {jsonl_file.name}")
+            
+        except Exception as e:
+            logger.error(f"Failed to import {jsonl_file}: {e}")
+            import traceback
+            logger.error(traceback.format_exc())
+    
+    return total_chunks
+
+def main():
+    """Main import function."""
+    logs_path = Path(LOGS_DIR)
+    
+    if not logs_path.exists():
+        logger.error(f"Logs directory not found: {LOGS_DIR}")
+        return
+    
+    # Find all project directories
+    project_dirs = [d for d in logs_path.iterdir() if d.is_dir()]
+    
+    if not project_dirs:
+        logger.warning("No project directories found")
+        return
+    
+    logger.info(f"Found {len(project_dirs)} projects to import")
+    
+    # Import each project
+    total_imported = 0
+    for project_dir in project_dirs:
+        # Create collection name from project path
+        collection_name = f"conv_{hashlib.md5(project_dir.name.encode()).hexdigest()[:8]}{collection_suffix}"
+        
+        logger.info(f"Importing project: {project_dir.name} -> {collection_name}")
+        chunks = import_project(project_dir, collection_name)
+        total_imported += chunks
+        logger.info(f"Imported {chunks} chunks from {project_dir.name}")
+    
+    logger.info(f"Import complete! Total chunks imported: {total_imported}")
+
+if __name__ == "__main__":
+    main()

package/scripts/import-recent-only.py

@@ -23,7 +23,11 @@ for file in os.listdir(project_path):
 print(f"Found {len(recent_files)} recent files to import")
 
 # Set environment variable
-os.environ["VOYAGE_KEY"] = "pa-wdTYGObaxhs-XFKX2r7WCczRwEVNb9eYMTSO3yrQhZI"
+# VOYAGE_KEY must be set as environment variable
+if not os.getenv("VOYAGE_KEY"):
+    print("Error: VOYAGE_KEY environment variable not set")
+    print("Please set: export VOYAGE_KEY='your-voyage-api-key'")
+    sys.exit(1)
 
 # Import the whole project (the script will handle individual files)
 os.system(f"python {import_script} {project_path}")

package/scripts/import-watcher.py

@@ -19,7 +19,7 @@ WATCH_DIR = os.getenv("WATCH_DIR", "/logs")
 STATE_FILE = os.getenv("STATE_FILE", "/config/imported-files.json")
 WATCH_INTERVAL = int(os.getenv("WATCH_INTERVAL", "60"))  # seconds
 IMPORT_DELAY = int(os.getenv("IMPORT_DELAY", "30"))  # Wait before importing new files
-IMPORTER_SCRIPT = "/scripts/import-conversations-voyage-streaming.py"
+IMPORTER_SCRIPT = "/scripts/import-conversations-unified.py"
 
 # Set up logging
 logging.basicConfig(