claude-self-reflect 2.5.18 → 2.6.0

package/scripts/force-metadata-recovery.py

@@ -0,0 +1,305 @@
+#!/usr/bin/env python3
+"""
+Force metadata recovery script for Claude Self-Reflect.
+Fixes conversations that were marked as updated but don't actually have metadata.
+This addresses the point ID mismatch bug in delta-metadata-update.py.
+"""
+
+import os
+import sys
+import json
+import hashlib
+import re
+import asyncio
+from datetime import datetime
+from typing import List, Dict, Any, Set, Optional
+import logging
+from pathlib import Path
+
+from qdrant_client import QdrantClient
+
+# Configuration
+QDRANT_URL = os.getenv("QDRANT_URL", "http://localhost:6333")
+LOGS_DIR = os.getenv("LOGS_DIR", os.path.expanduser("~/.claude/projects"))
+DRY_RUN = os.getenv("DRY_RUN", "false").lower() == "true"
+BATCH_SIZE = int(os.getenv("BATCH_SIZE", "100"))
+
+# Set up logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+# Initialize Qdrant client
+client = QdrantClient(url=QDRANT_URL, timeout=30)
+
+def normalize_path(path: str) -> str:
+    """Normalize file paths for consistency."""
+    if not path:
+        return ""
+    path = path.replace("/Users/", "~/").replace("\\Users\\", "~\\")
+    path = path.replace("\\", "/")
+    path = re.sub(r'/+', '/', path)
+    return path
+
+def extract_concepts(text: str) -> Set[str]:
+    """Extract high-level concepts from text."""
+    concepts = set()
+    
+    concept_patterns = {
+        'security': r'(security|vulnerability|CVE|injection|auth)',
+        'docker': r'(docker|container|compose|kubernetes)',
+        'testing': r'(test|pytest|unittest|coverage)',
+        'api': r'(API|REST|GraphQL|endpoint)',
+        'database': r'(database|SQL|query|migration|qdrant)',
+        'debugging': r'(debug|error|exception|traceback)',
+        'git': r'(git|commit|branch|merge|pull request)',
+        'mcp': r'(MCP|claude-self-reflect|tool|agent)',
+        'embeddings': r'(embedding|vector|semantic|similarity)',
+    }
+    
+    text_lower = text.lower()
+    for concept, pattern in concept_patterns.items():
+        if re.search(pattern, text_lower, re.IGNORECASE):
+            concepts.add(concept)
+    
+    return concepts
+
+def extract_metadata_from_jsonl(jsonl_path: str) -> Dict[str, Any]:
+    """Extract metadata from a JSONL conversation file."""
+    metadata = {
+        "files_analyzed": [],
+        "files_edited": [],
+        "tools_used": set(),
+        "concepts": set(),
+        "text_sample": ""
+    }
+    
+    try:
+        with open(jsonl_path, 'r', encoding='utf-8') as f:
+            line_count = 0
+            for line in f:
+                line_count += 1
+                if line_count > 200:  # Limit processing
+                    break
+                    
+                if not line.strip():
+                    continue
+                    
+                try:
+                    data = json.loads(line)
+                    if 'message' in data and data['message']:
+                        msg = data['message']
+                        
+                        # Extract text for concept analysis
+                        if msg.get('content'):
+                            if isinstance(msg['content'], str):
+                                metadata['text_sample'] += msg['content'][:500] + "\n"
+                            
+                        # Extract tool usage
+                        if msg.get('role') == 'assistant' and msg.get('content'):
+                            content = msg['content']
+                            if isinstance(content, list):
+                                for item in content:
+                                    if isinstance(item, dict) and item.get('type') == 'tool_use':
+                                        tool_name = item.get('name', '')
+                                        metadata['tools_used'].add(tool_name)
+                                        
+                                        inputs = item.get('input', {})
+                                        
+                                        if tool_name == 'Read' and 'file_path' in inputs:
+                                            metadata['files_analyzed'].append(
+                                                normalize_path(inputs['file_path'])
+                                            )
+                                        elif tool_name in ['Edit', 'Write'] and 'file_path' in inputs:
+                                            metadata['files_edited'].append(
+                                                normalize_path(inputs['file_path'])
+                                            )
+                                            
+                except json.JSONDecodeError:
+                    continue
+                    
+    except Exception as e:
+        logger.error(f"Error reading {jsonl_path}: {e}")
+    
+    # Extract concepts from collected text
+    if metadata['text_sample']:
+        metadata['concepts'] = extract_concepts(metadata['text_sample'][:5000])
+    
+    # Convert sets to lists and limit
+    metadata['tools_used'] = list(metadata['tools_used'])[:20]
+    metadata['concepts'] = list(metadata['concepts'])[:15]
+    metadata['files_analyzed'] = list(set(metadata['files_analyzed']))[:20]
+    metadata['files_edited'] = list(set(metadata['files_edited']))[:10]
+    
+    del metadata['text_sample']  # Don't store in Qdrant
+    
+    return metadata
+
+async def find_conversations_without_metadata(collection_name: str) -> List[str]:
+    """Find all unique conversation IDs that don't have metadata."""
+    conversations_without_metadata = set()
+    
+    offset = None
+    total_checked = 0
+    
+    while True:
+        points, next_offset = client.scroll(
+            collection_name=collection_name,
+            limit=BATCH_SIZE,
+            offset=offset,
+            with_payload=True,
+            with_vectors=False
+        )
+        
+        if not points:
+            break
+            
+        for point in points:
+            # Check if metadata is missing
+            if not point.payload.get('concepts') or not point.payload.get('has_file_metadata'):
+                conv_id = point.payload.get('conversation_id')
+                if conv_id:
+                    conversations_without_metadata.add(conv_id)
+        
+        total_checked += len(points)
+        offset = next_offset
+        
+        if offset is None:
+            break
+    
+    logger.info(f"  Checked {total_checked} points, found {len(conversations_without_metadata)} conversations without metadata")
+    return list(conversations_without_metadata)
+
+async def update_conversation_points(collection_name: str, conversation_id: str, metadata: Dict[str, Any]) -> int:
+    """Update all points for a conversation with metadata."""
+    updated_count = 0
+    
+    # Get all points in the collection
+    offset = None
+    while True:
+        points, next_offset = client.scroll(
+            collection_name=collection_name,
+            limit=BATCH_SIZE,
+            offset=offset,
+            with_payload=True,
+            with_vectors=False
+        )
+        
+        if not points:
+            break
+        
+        # Find and update points for this conversation
+        for point in points:
+            if point.payload.get('conversation_id') == conversation_id:
+                if not DRY_RUN:
+                    # Merge metadata with existing payload
+                    updated_payload = {**point.payload, **metadata}
+                    updated_payload['has_file_metadata'] = True
+                    updated_payload['metadata_updated_at'] = datetime.now().isoformat()
+                    
+                    client.set_payload(
+                        collection_name=collection_name,
+                        payload=updated_payload,
+                        points=[point.id],
+                        wait=False
+                    )
+                
+                updated_count += 1
+        
+        offset = next_offset
+        if offset is None:
+            break
+    
+    return updated_count
+
+async def process_collection(collection_name: str):
+    """Process a single collection to add missing metadata."""
+    logger.info(f"\nProcessing collection: {collection_name}")
+    
+    # Find conversations without metadata
+    conversations_without_metadata = await find_conversations_without_metadata(collection_name)
+    
+    if not conversations_without_metadata:
+        logger.info(f"  ✓ All conversations have metadata")
+        return 0
+    
+    logger.info(f"  Found {len(conversations_without_metadata)} conversations needing metadata")
+    
+    # Process each conversation
+    success_count = 0
+    failed_count = 0
+    
+    for conv_id in conversations_without_metadata[:10]:  # Limit for testing
+        # Find the JSONL file
+        jsonl_pattern = f"**/{conv_id}.jsonl"
+        jsonl_files = list(Path(LOGS_DIR).glob(jsonl_pattern))
+        
+        if not jsonl_files:
+            logger.warning(f"  Cannot find JSONL for {conv_id}")
+            failed_count += 1
+            continue
+        
+        jsonl_file = jsonl_files[0]
+        logger.info(f"  Processing {conv_id}")
+        
+        # Extract metadata
+        metadata = extract_metadata_from_jsonl(str(jsonl_file))
+        
+        if not metadata['concepts'] and not metadata['files_analyzed']:
+            logger.warning(f"    No metadata extracted from {conv_id}")
+            failed_count += 1
+            continue
+        
+        # Update points
+        updated_points = await update_conversation_points(collection_name, conv_id, metadata)
+        
+        if updated_points > 0:
+            logger.info(f"    ✓ Updated {updated_points} points with {len(metadata['concepts'])} concepts")
+            success_count += 1
+        else:
+            logger.warning(f"    No points updated for {conv_id}")
+            failed_count += 1
+    
+    logger.info(f"  Collection complete: {success_count} fixed, {failed_count} failed")
+    return success_count
+
+async def main():
+    """Main recovery process."""
+    logger.info("=== Force Metadata Recovery ===")
+    logger.info(f"Qdrant URL: {QDRANT_URL}")
+    logger.info(f"Dry run: {DRY_RUN}")
+    
+    # Get all collections
+    collections = client.get_collections().collections
+    
+    # Focus on collections with potential issues
+    priority_collections = []
+    other_collections = []
+    
+    for collection in collections:
+        name = collection.name
+        if name.startswith('conv_'):
+            other_collections.append(name)
+    
+    logger.info(f"Found {len(priority_collections)} priority collections")
+    logger.info(f"Found {len(other_collections)} other collections")
+    
+    # Process priority collections first
+    total_fixed = 0
+    
+    for collection_name in priority_collections:
+        fixed = await process_collection(collection_name)
+        total_fixed += fixed
+    
+    # Process a sample of other collections
+    for collection_name in other_collections[:5]:
+        fixed = await process_collection(collection_name)
+        total_fixed += fixed
+    
+    logger.info(f"\n=== Recovery Complete ===")
+    logger.info(f"Total conversations fixed: {total_fixed}")
+    
+    if DRY_RUN:
+        logger.info("This was a DRY RUN - no actual updates were made")
+
+if __name__ == "__main__":
+    asyncio.run(main())

package/scripts/import-conversations-unified.py

@@ -42,6 +42,20 @@ PREFER_LOCAL_EMBEDDINGS = os.getenv("PREFER_LOCAL_EMBEDDINGS", "false").lower()
 VOYAGE_API_KEY = os.getenv("VOYAGE_KEY")
 CURRENT_METADATA_VERSION = 2  # Version 2: Added tool output extraction
 
+# Token limit configuration for Voyage AI
+MAX_TOKENS_PER_BATCH = int(os.getenv("MAX_TOKENS_PER_BATCH", "100000"))  # Safe limit (120k - 20k buffer)
+if MAX_TOKENS_PER_BATCH > 120000 or MAX_TOKENS_PER_BATCH < 1000:
+    logger.warning(f"MAX_TOKENS_PER_BATCH={MAX_TOKENS_PER_BATCH} outside safe range [1000, 120000], using 100000")
+    MAX_TOKENS_PER_BATCH = 100000
+
+TOKEN_ESTIMATION_RATIO = int(os.getenv("TOKEN_ESTIMATION_RATIO", "3"))  # chars per token estimate
+if TOKEN_ESTIMATION_RATIO < 2 or TOKEN_ESTIMATION_RATIO > 10:
+    logger.warning(f"TOKEN_ESTIMATION_RATIO={TOKEN_ESTIMATION_RATIO} outside normal range [2, 10], using 3")
+    TOKEN_ESTIMATION_RATIO = 3
+
+USE_TOKEN_AWARE_BATCHING = os.getenv("USE_TOKEN_AWARE_BATCHING", "true").lower() == "true"
+MAX_RECURSION_DEPTH = 10  # Maximum depth for recursive chunk splitting
+
 # Set up logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 logger = logging.getLogger(__name__)
@@ -381,6 +395,38 @@ def log_retry_state(retry_state):
 def embed_with_backoff(**kwargs):
     return voyage_client.embed(**kwargs)
 
+def estimate_tokens(text: str) -> int:
+    """Estimate token count for text with content-aware heuristics.
+    Base estimate uses TOKEN_ESTIMATION_RATIO, adjusted for content type.
+    """
+    # Base estimate
+    base_tokens = len(text) // TOKEN_ESTIMATION_RATIO
+    
+    # Adjust for code/JSON content (typically more tokens per char)
+    # Count indicators of structured content
+    structure_indicators = text.count('{') + text.count('[') + text.count('```')
+    if structure_indicators > 10:  # Likely JSON/code
+        base_tokens = int(base_tokens * 1.3)
+    
+    # Add 10% safety margin
+    return int(base_tokens * 1.1)
+
+def extract_message_content(msg: Dict[str, Any]) -> str:
+    """Extract text content from a message."""
+    content = msg.get("content", "")
+    
+    if isinstance(content, list):
+        # Handle structured content
+        text_parts = []
+        for item in content:
+            if isinstance(item, dict) and item.get("type") == "text":
+                text_parts.append(item.get("text", ""))
+            elif isinstance(item, str):
+                text_parts.append(item)
+        content = " ".join(text_parts)
+    
+    return content
+
 def generate_embeddings(texts: List[str]) -> List[List[float]]:
     """Generate embeddings for a list of texts."""
     if PREFER_LOCAL_EMBEDDINGS or not VOYAGE_API_KEY:
@@ -432,6 +478,125 @@ def chunk_conversation(messages: List[Dict[str, Any]], chunk_size: int = 10) ->
     
     return chunks
 
+def split_large_chunk(chunk: Dict[str, Any], max_tokens: int, depth: int = 0) -> List[Dict[str, Any]]:
+    """Split a large chunk into smaller pieces that fit token limit."""
+    # Check recursion depth to prevent stack overflow
+    if depth >= MAX_RECURSION_DEPTH:
+        logger.error(f"Max recursion depth {MAX_RECURSION_DEPTH} reached while splitting chunk")
+        # Force truncate as last resort
+        max_chars = max_tokens * TOKEN_ESTIMATION_RATIO
+        chunk["text"] = chunk["text"][:max_chars] + "\n[TRUNCATED - MAX DEPTH REACHED]"
+        chunk["was_truncated"] = True
+        return [chunk]
+    
+    text = chunk["text"]
+    messages = chunk["messages"]
+    
+    # First, check if we can split by messages
+    if len(messages) > 1:
+        # Try splitting messages into smaller groups
+        mid = len(messages) // 2
+        chunk1_messages = messages[:mid]
+        chunk2_messages = messages[mid:]
+        
+        # Recreate text for each split
+        texts1 = []
+        texts2 = []
+        
+        for msg in chunk1_messages:
+            role = msg.get("role", "unknown")
+            content = extract_message_content(msg)
+            if content:
+                texts1.append(f"{role.upper()}: {content}")
+        
+        for msg in chunk2_messages:
+            role = msg.get("role", "unknown")
+            content = extract_message_content(msg)
+            if content:
+                texts2.append(f"{role.upper()}: {content}")
+        
+        split_chunks = []
+        if texts1:
+            split_chunks.append({
+                "text": "\n".join(texts1),
+                "messages": chunk1_messages,
+                "chunk_index": f"{chunk['chunk_index']}_a",
+                "start_role": chunk["start_role"]
+            })
+        if texts2:
+            split_chunks.append({
+                "text": "\n".join(texts2),
+                "messages": chunk2_messages,
+                "chunk_index": f"{chunk['chunk_index']}_b",
+                "start_role": chunk2_messages[0].get("role", "unknown") if chunk2_messages else "unknown"
+            })
+        
+        # Recursively split if still too large
+        result = []
+        for split_chunk in split_chunks:
+            if estimate_tokens(split_chunk["text"]) > max_tokens:
+                result.extend(split_large_chunk(split_chunk, max_tokens, depth + 1))
+            else:
+                result.append(split_chunk)
+        return result
+    else:
+        # Single message too large - truncate with warning
+        max_chars = max_tokens * TOKEN_ESTIMATION_RATIO
+        if len(text) > max_chars:
+            truncated_size = len(text) - max_chars
+            logger.warning(f"Single message exceeds token limit, truncating {truncated_size} chars from {len(text)} total")
+            chunk["text"] = text[:max_chars] + f"\n[TRUNCATED {truncated_size} CHARS]"
+            chunk["was_truncated"] = True
+            chunk["original_size"] = len(text)
+        return [chunk]
+
+def create_token_aware_batches(chunks: List[Dict[str, Any]], max_tokens: int = MAX_TOKENS_PER_BATCH) -> List[List[Dict[str, Any]]]:
+    """Create batches that respect token limits."""
+    if not USE_TOKEN_AWARE_BATCHING:
+        # Fall back to old batching method
+        batches = []
+        for i in range(0, len(chunks), BATCH_SIZE):
+            batches.append(chunks[i:i + BATCH_SIZE])
+        return batches
+    
+    batches = []
+    current_batch = []
+    current_tokens = 0
+    
+    for chunk in chunks:
+        chunk_tokens = estimate_tokens(chunk["text"])
+        
+        # If single chunk exceeds limit, split it
+        if chunk_tokens > max_tokens:
+            logger.warning(f"Chunk with {chunk_tokens} estimated tokens exceeds limit of {max_tokens}, splitting...")
+            split_chunks = split_large_chunk(chunk, max_tokens)
+            for split_chunk in split_chunks:
+                split_tokens = estimate_tokens(split_chunk["text"])
+                if split_tokens > max_tokens:
+                    logger.error(f"Split chunk still exceeds limit: {split_tokens} tokens")
+                batches.append([split_chunk])
+        # If adding chunk would exceed limit, start new batch
+        elif current_tokens + chunk_tokens > max_tokens:
+            if current_batch:
+                batches.append(current_batch)
+            current_batch = [chunk]
+            current_tokens = chunk_tokens
+        else:
+            current_batch.append(chunk)
+            current_tokens += chunk_tokens
+    
+    if current_batch:
+        batches.append(current_batch)
+    
+    # Log batch statistics
+    if batches:
+        batch_sizes = [len(batch) for batch in batches]
+        batch_tokens = [sum(estimate_tokens(chunk["text"]) for chunk in batch) for batch in batches]
+        logger.debug(f"Created {len(batches)} batches, chunk counts: min={min(batch_sizes)}, max={max(batch_sizes)}, "
+                    f"estimated tokens: min={min(batch_tokens)}, max={max(batch_tokens)}, avg={sum(batch_tokens)//len(batches)}")
+    
+    return batches
+
 def import_project(project_path: Path, collection_name: str, state: dict) -> int:
     """Import all conversations from a project."""
     jsonl_files = list(project_path.glob("*.jsonl"))
@@ -524,11 +689,17 @@ def import_project(project_path: Path, collection_name: str, state: dict) -> int
             if not chunks:
                 continue
             
-            # Process in batches
-            for batch_start in range(0, len(chunks), BATCH_SIZE):
-                batch = chunks[batch_start:batch_start + BATCH_SIZE]
+            # Process in batches (token-aware if enabled)
+            token_aware_batches = create_token_aware_batches(chunks)
+            
+            for batch_idx, batch in enumerate(token_aware_batches):
                 texts = [chunk["text"] for chunk in batch]
                 
+                # Log batch info for debugging
+                if USE_TOKEN_AWARE_BATCHING:
+                    total_tokens = sum(estimate_tokens(text) for text in texts)
+                    logger.debug(f"Batch {batch_idx + 1}/{len(token_aware_batches)}: {len(texts)} chunks, ~{total_tokens} estimated tokens")
+                
                 # Generate embeddings
                 embeddings = generate_embeddings(texts)