claude-self-reflect 7.1.9 → 7.1.11

package/docs/design/recover_batch_results.py

@@ -0,0 +1,287 @@
+#!/usr/bin/env python3
+"""
+Recover batch results and complete the Qdrant import.
+
+Retrieves narratives from completed batches and imports to Qdrant.
+"""
+
+import os
+import sys
+import json
+from pathlib import Path
+from dotenv import load_dotenv
+import time
+
+load_dotenv()
+
+sys.path.insert(0, str(Path(__file__).parent.parent.parent))
+
+import anthropic
+from qdrant_client import QdrantClient
+from qdrant_client.models import PointStruct
+
+# Import FastEmbed
+try:
+    from fastembed import TextEmbedding
+    FASTEMBED_AVAILABLE = True
+except ImportError:
+    FASTEMBED_AVAILABLE = False
+    print("⚠️  FastEmbed not available")
+    sys.exit(1)
+
+
+# Batch IDs from dashboard (in chronological order)
+COMPLETED_BATCHES = [
+    {
+        'batch_id': 'msgbatch_01ATPhpjCw1gqPisHUgoPnab',
+        'project': 'address-book-fix',
+        'count': 1
+    },
+    {
+        'batch_id': 'msgbatch_016g8zHtH7or7DtJu3ZzAczS',
+        'project': 'anukruti',
+        'count': 5
+    },
+    {
+        'batch_id': 'msgbatch_01QCwhFw9DYDJ8uPjYsHg8Xu',
+        'project': 'buyindian',
+        'count': 5
+    },
+    {
+        'batch_id': 'msgbatch_01WVbb5X2xYwuzzgEdqVicZJ',
+        'project': 'procsolve-website',  # or cc-enhance
+        'count': 2
+    },
+    {
+        'batch_id': 'msgbatch_01EemyvChmnShYAuJix7m1As',
+        'project': 'claude-self-reflect',
+        'count': 36
+    }
+]
+
+
+def get_embedding(text: str, embedding_model) -> list:
+    """Generate embedding for text."""
+    embeddings = list(embedding_model.embed([text]))
+    return embeddings[0].tolist()
+
+
+def retrieve_batch_narratives(client: anthropic.Anthropic, batch_id: str):
+    """Retrieve narratives from a completed batch."""
+
+    print(f"\n🔄 Retrieving batch {batch_id}...")
+
+    try:
+        # Get batch results
+        results_response = client.messages.batches.results(batch_id)
+
+        narratives = {}
+        total_cost = 0.0
+        total_input = 0
+        total_output = 0
+
+        for result_item in results_response:
+            conv_id = result_item.custom_id
+
+            if result_item.result.type == "succeeded":
+                message = result_item.result.message
+
+                # Extract narrative
+                narrative = ""
+                for block in message.content:
+                    if hasattr(block, 'text'):
+                        narrative += block.text
+
+                narratives[conv_id] = narrative
+
+                # Track usage
+                input_tokens = message.usage.input_tokens
+                output_tokens = message.usage.output_tokens
+                cost = (input_tokens * 3 + output_tokens * 15) / 1_000_000
+
+                total_input += input_tokens
+                total_output += output_tokens
+                total_cost += cost
+            else:
+                print(f"  ❌ Error for {conv_id}: {result_item.result.error}")
+
+        print(f"  ✅ Retrieved {len(narratives)} narratives")
+        print(f"  📊 Tokens: {total_input} input, {total_output} output")
+        print(f"  💰 Cost: ${total_cost:.4f}")
+
+        return narratives, total_cost
+
+    except Exception as e:
+        print(f"  ❌ Failed to retrieve batch: {e}")
+        return {}, 0.0
+
+
+def load_conversation_data(project_dir: Path):
+    """Load V3 extraction results and metadata for a project."""
+
+    conversations = {}
+
+    # Import metadata extraction functions
+    import importlib.util
+    delta_metadata_path = Path(__file__).parent.parent.parent / "src" / "runtime" / "delta-metadata-update.py"
+    spec = importlib.util.spec_from_file_location("delta_metadata_update", delta_metadata_path)
+    delta_metadata_update = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(delta_metadata_update)
+    extract_tool_usage_from_jsonl = delta_metadata_update.extract_tool_usage_from_jsonl
+    extract_concepts = delta_metadata_update.extract_concepts
+
+    from docs.design.extract_events_v3 import extract_events_v3
+
+    for jsonl_file in project_dir.glob("*.jsonl"):
+        conv_id = jsonl_file.stem
+
+        # Extract metadata
+        tool_usage = extract_tool_usage_from_jsonl(str(jsonl_file))
+
+        # Read messages for V3 extraction
+        messages = []
+        conversation_text = ""
+        with open(jsonl_file) as f:
+            for line in f:
+                if line.strip():
+                    msg = json.loads(line)
+                    messages.append(msg)
+
+                    if 'message' in msg and msg['message']:
+                        content = msg['message'].get('content', '')
+                        if isinstance(content, str):
+                            conversation_text += content + "\n"
+                        elif isinstance(content, list):
+                            for item in content:
+                                if isinstance(item, dict) and item.get('text'):
+                                    conversation_text += item['text'] + "\n"
+
+        # Extract concepts
+        concepts = extract_concepts(conversation_text[:10000], tool_usage)
+
+        # Build metadata dict
+        metadata = {
+            'tool_usage': tool_usage,
+            'concepts': concepts
+        }
+
+        # V3 extraction WITH metadata
+        result = extract_events_v3(messages, metadata=metadata)
+
+        conversations[conv_id] = result
+
+    return conversations
+
+
+def main():
+    """Recover and import batch results."""
+
+    print(f"\n{'='*80}")
+    print(f"BATCH RESULTS RECOVERY & QDRANT IMPORT")
+    print(f"{'='*80}\n")
+
+    # Initialize clients
+    print("🔧 Initializing clients...")
+    anthropic_client = anthropic.Anthropic(api_key=os.getenv("ANTHROPIC_API_KEY"))
+    qdrant_client = QdrantClient(url=os.getenv("QDRANT_URL", "http://localhost:6333"))
+    embedding_model = TextEmbedding(model_name="sentence-transformers/all-MiniLM-L6-v2")
+    print("  ✅ Clients initialized")
+
+    # Collection should already exist
+    collection_name = "v3_all_projects"
+
+    # Retrieve all batch results
+    all_narratives = {}
+    grand_total_cost = 0.0
+
+    for batch_info in COMPLETED_BATCHES:
+        narratives, cost = retrieve_batch_narratives(
+            anthropic_client,
+            batch_info['batch_id']
+        )
+
+        # Tag narratives with project
+        for conv_id, narrative in narratives.items():
+            all_narratives[conv_id] = {
+                'narrative': narrative,
+                'project': batch_info['project']
+            }
+
+        grand_total_cost += cost
+
+    print(f"\n📊 Total narratives retrieved: {len(all_narratives)}")
+    print(f"💰 Total cost: ${grand_total_cost:.4f}")
+
+    # Load conversation data and create points
+    print(f"\n🔄 Loading conversation data and creating points...")
+
+    projects_dir = Path.home() / ".claude/projects"
+    all_points = []
+
+    for batch_info in COMPLETED_BATCHES:
+        project_name = batch_info['project']
+
+        # Find project directory
+        project_dirs = list(projects_dir.glob(f"*{project_name}"))
+        if not project_dirs:
+            print(f"  ⚠️  Project directory not found for {project_name}")
+            continue
+
+        project_dir = project_dirs[0]
+        print(f"\n  Processing {project_name}...")
+
+        # Load conversation data
+        conversations = load_conversation_data(project_dir)
+
+        # Create points
+        for conv_id, result in conversations.items():
+            if conv_id not in all_narratives:
+                print(f"    ⚠️  No narrative for {conv_id}")
+                continue
+
+            narrative = all_narratives[conv_id]['narrative']
+
+            # Generate embedding
+            embedding = get_embedding(narrative, embedding_model)
+
+            # Create point
+            point = PointStruct(
+                id=conv_id,
+                vector=embedding,
+                payload={
+                    "conversation_id": conv_id,
+                    "project": project_name,
+                    "narrative": narrative,
+                    "search_index": result['search_index'],
+                    "context_cache": result['context_cache'],
+                    "signature": result['signature'],
+                    "timestamp": time.time(),
+                    "extraction_stats": result['stats']
+                }
+            )
+
+            all_points.append(point)
+            print(f"    ✅ {conv_id}")
+
+    # Import to Qdrant
+    print(f"\n🔄 Importing {len(all_points)} points to Qdrant...")
+
+    batch_size = 100
+    for i in range(0, len(all_points), batch_size):
+        batch = all_points[i:i+batch_size]
+        qdrant_client.upsert(
+            collection_name=collection_name,
+            points=batch
+        )
+        print(f"  ✅ Imported batch {i//batch_size + 1}: {len(batch)} points")
+
+    # Verify
+    collection_info = qdrant_client.get_collection(collection_name)
+    print(f"\n✅ RECOVERY COMPLETE!")
+    print(f"   Collection: {collection_name}")
+    print(f"   Total points: {collection_info.points_count}")
+    print(f"   Total cost: ${grand_total_cost:.4f}")
+
+
+if __name__ == "__main__":
+    main()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-self-reflect",
-  "version": "7.1.9",
+  "version": "7.1.11",
   "description": "Give Claude perfect memory of all your conversations - Installation wizard for Python MCP server",
   "keywords": [
     "claude",
@@ -39,6 +39,7 @@
     "scripts/auto-migrate.cjs",
     "scripts/migrate-to-unified-state.py",
     "scripts/csr-status",
+    "scripts/ralph/**",
     "mcp-server/src/**/*.py",
     "mcp-server/pyproject.toml",
     "mcp-server/run-mcp.sh",
@@ -52,6 +53,9 @@
     "config/qdrant-config.yaml",
     "docker-compose.yaml",
     "Dockerfile.*",
+    "docs/design/**/*.py",
+    "docs/design/**/*.md",
+    "docs/design/conversation-analyzer/**",
     ".env.example",
     "README.md",
     "LICENSE"

package/scripts/ralph/backup_and_restore.sh

@@ -0,0 +1,309 @@
+#!/bin/bash
+# =============================================================================
+# Ralph Memory Integration - Backup and Restore Script
+# =============================================================================
+# Usage:
+#   ./backup_and_restore.sh backup           # Create full backup
+#   ./backup_and_restore.sh restore <dir>    # Restore from backup directory
+#   ./backup_and_restore.sh verify <dir>     # Verify backup integrity
+#   ./backup_and_restore.sh list             # List available backups
+# =============================================================================
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+BACKUP_BASE="$HOME/.claude-self-reflect/backups"
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
+log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
+log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
+
+# =============================================================================
+# BACKUP FUNCTION
+# =============================================================================
+do_backup() {
+    local BACKUP_DIR="$BACKUP_BASE/$(date +%Y%m%d_%H%M%S)_pre_ralph_memory"
+    mkdir -p "$BACKUP_DIR"
+
+    log_info "Creating backup at: $BACKUP_DIR"
+
+    # 1. Stop services for consistent backup
+    log_info "Stopping services for consistent backup..."
+    docker stop claude-reflection-batch-watcher claude-reflection-batch-monitor 2>/dev/null || true
+    sleep 2
+
+    # 2. Backup Qdrant data volume
+    log_info "Backing up Qdrant data volume..."
+    if docker volume inspect qdrant_data > /dev/null 2>&1; then
+        docker run --rm \
+            -v qdrant_data:/data:ro \
+            -v "$BACKUP_DIR":/backup \
+            alpine tar czf /backup/qdrant_data.tar.gz -C /data .
+        log_info "Qdrant backup: $(du -h "$BACKUP_DIR/qdrant_data.tar.gz" | cut -f1)"
+    else
+        log_warn "Qdrant volume not found, skipping"
+    fi
+
+    # 3. Backup CSR config directory
+    log_info "Backing up CSR config..."
+    if [ -d "$HOME/.claude-self-reflect/config" ]; then
+        tar czf "$BACKUP_DIR/csr_config.tar.gz" -C "$HOME/.claude-self-reflect" config
+    else
+        log_warn "CSR config not found, skipping"
+    fi
+
+    # 4. Backup batch queue
+    if [ -d "$HOME/.claude-self-reflect/batch_queue" ]; then
+        tar czf "$BACKUP_DIR/csr_batch_queue.tar.gz" -C "$HOME/.claude-self-reflect" batch_queue
+    fi
+
+    # 5. Backup batch state
+    if [ -d "$HOME/.claude-self-reflect/batch_state" ]; then
+        tar czf "$BACKUP_DIR/csr_batch_state.tar.gz" -C "$HOME/.claude-self-reflect" batch_state
+    fi
+
+    # 6. Save git state
+    log_info "Saving git state..."
+    cd "$PROJECT_ROOT"
+    echo "$(git rev-parse HEAD)" > "$BACKUP_DIR/git_head.txt"
+    echo "$(git branch --show-current)" > "$BACKUP_DIR/git_branch.txt"
+    git diff > "$BACKUP_DIR/git_diff.patch" 2>/dev/null || true
+    git diff --cached > "$BACKUP_DIR/git_staged.patch" 2>/dev/null || true
+
+    # 7. Restart services
+    log_info "Restarting services..."
+    docker start claude-reflection-batch-watcher claude-reflection-batch-monitor 2>/dev/null || true
+
+    # 8. Create manifest
+    cat > "$BACKUP_DIR/manifest.json" << EOF
+{
+    "created": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
+    "git_commit": "$(cat "$BACKUP_DIR/git_head.txt")",
+    "git_branch": "$(cat "$BACKUP_DIR/git_branch.txt")",
+    "qdrant_backup": $([ -f "$BACKUP_DIR/qdrant_data.tar.gz" ] && echo "true" || echo "false"),
+    "config_backup": $([ -f "$BACKUP_DIR/csr_config.tar.gz" ] && echo "true" || echo "false"),
+    "project_root": "$PROJECT_ROOT"
+}
+EOF
+
+    log_info "Backup complete!"
+    echo ""
+    echo "Backup directory: $BACKUP_DIR"
+    echo "Files:"
+    ls -lh "$BACKUP_DIR"
+    echo ""
+    echo "To restore: $0 restore $BACKUP_DIR"
+}
+
+# =============================================================================
+# RESTORE FUNCTION
+# =============================================================================
+do_restore() {
+    local BACKUP_DIR="$1"
+
+    if [ -z "$BACKUP_DIR" ]; then
+        log_error "Usage: $0 restore <backup_directory>"
+        exit 1
+    fi
+
+    if [ ! -d "$BACKUP_DIR" ]; then
+        log_error "Backup directory not found: $BACKUP_DIR"
+        exit 1
+    fi
+
+    if [ ! -f "$BACKUP_DIR/manifest.json" ]; then
+        log_error "Invalid backup: manifest.json not found"
+        exit 1
+    fi
+
+    log_warn "This will restore system state from backup."
+    log_warn "Current changes will be lost!"
+    echo ""
+    read -p "Are you sure? (type 'yes' to confirm): " confirm
+    if [ "$confirm" != "yes" ]; then
+        log_info "Restore cancelled"
+        exit 0
+    fi
+
+    echo ""
+    log_info "Starting restore from: $BACKUP_DIR"
+
+    # 1. Stop all services
+    log_info "Stopping all services..."
+    docker stop claude-reflection-batch-watcher claude-reflection-batch-monitor claude-reflection-qdrant 2>/dev/null || true
+    sleep 3
+
+    # 2. Restore Qdrant data
+    if [ -f "$BACKUP_DIR/qdrant_data.tar.gz" ]; then
+        log_info "Restoring Qdrant data..."
+        docker run --rm \
+            -v qdrant_data:/data \
+            -v "$BACKUP_DIR":/backup \
+            alpine sh -c "rm -rf /data/* && tar xzf /backup/qdrant_data.tar.gz -C /data"
+    fi
+
+    # 3. Restore CSR config
+    if [ -f "$BACKUP_DIR/csr_config.tar.gz" ]; then
+        log_info "Restoring CSR config..."
+        rm -rf "$HOME/.claude-self-reflect/config"
+        tar xzf "$BACKUP_DIR/csr_config.tar.gz" -C "$HOME/.claude-self-reflect/"
+    fi
+
+    # 4. Restore batch queue
+    if [ -f "$BACKUP_DIR/csr_batch_queue.tar.gz" ]; then
+        rm -rf "$HOME/.claude-self-reflect/batch_queue"
+        tar xzf "$BACKUP_DIR/csr_batch_queue.tar.gz" -C "$HOME/.claude-self-reflect/"
+    fi
+
+    # 5. Restore batch state
+    if [ -f "$BACKUP_DIR/csr_batch_state.tar.gz" ]; then
+        rm -rf "$HOME/.claude-self-reflect/batch_state"
+        tar xzf "$BACKUP_DIR/csr_batch_state.tar.gz" -C "$HOME/.claude-self-reflect/"
+    fi
+
+    # 6. Restore git state
+    log_info "Restoring git state..."
+    cd "$PROJECT_ROOT"
+    ORIGINAL_COMMIT=$(cat "$BACKUP_DIR/git_head.txt")
+    git reset --hard "$ORIGINAL_COMMIT"
+
+    # 7. Restart services
+    log_info "Restarting services..."
+    docker start claude-reflection-qdrant 2>/dev/null || true
+    sleep 5
+    docker start claude-reflection-batch-watcher claude-reflection-batch-monitor 2>/dev/null || true
+
+    log_info "Restore complete!"
+    echo ""
+    echo "Verifying services..."
+    docker ps --filter "name=claude" --format "table {{.Names}}\t{{.Status}}"
+}
+
+# =============================================================================
+# VERIFY FUNCTION
+# =============================================================================
+do_verify() {
+    local BACKUP_DIR="$1"
+
+    if [ -z "$BACKUP_DIR" ]; then
+        log_error "Usage: $0 verify <backup_directory>"
+        exit 1
+    fi
+
+    if [ ! -d "$BACKUP_DIR" ]; then
+        log_error "Backup directory not found: $BACKUP_DIR"
+        exit 1
+    fi
+
+    echo "=========================================="
+    echo "Verifying backup: $BACKUP_DIR"
+    echo "=========================================="
+    echo ""
+
+    local ALL_OK=true
+
+    # Check manifest
+    if [ -f "$BACKUP_DIR/manifest.json" ]; then
+        log_info "✓ Manifest found"
+        cat "$BACKUP_DIR/manifest.json" | python3 -m json.tool 2>/dev/null || log_warn "Manifest is not valid JSON"
+    else
+        log_error "✗ Manifest missing"
+        ALL_OK=false
+    fi
+
+    # Check Qdrant backup
+    if [ -f "$BACKUP_DIR/qdrant_data.tar.gz" ]; then
+        local SIZE=$(du -h "$BACKUP_DIR/qdrant_data.tar.gz" | cut -f1)
+        log_info "✓ Qdrant backup ($SIZE)"
+        # Verify tar integrity
+        tar tzf "$BACKUP_DIR/qdrant_data.tar.gz" > /dev/null 2>&1 && log_info "  ✓ Archive integrity OK" || { log_error "  ✗ Archive corrupted"; ALL_OK=false; }
+    else
+        log_warn "✗ Qdrant backup missing"
+    fi
+
+    # Check config backup
+    if [ -f "$BACKUP_DIR/csr_config.tar.gz" ]; then
+        local SIZE=$(du -h "$BACKUP_DIR/csr_config.tar.gz" | cut -f1)
+        log_info "✓ Config backup ($SIZE)"
+    else
+        log_warn "○ Config backup missing (optional)"
+    fi
+
+    # Check git state
+    if [ -f "$BACKUP_DIR/git_head.txt" ]; then
+        log_info "✓ Git state saved: $(cat "$BACKUP_DIR/git_head.txt" | head -c 8)..."
+    else
+        log_error "✗ Git state missing"
+        ALL_OK=false
+    fi
+
+    echo ""
+    if $ALL_OK; then
+        log_info "Backup verification: PASSED"
+    else
+        log_error "Backup verification: FAILED"
+        exit 1
+    fi
+}
+
+# =============================================================================
+# LIST FUNCTION
+# =============================================================================
+do_list() {
+    echo "=========================================="
+    echo "Available Backups"
+    echo "=========================================="
+
+    if [ ! -d "$BACKUP_BASE" ]; then
+        log_info "No backups found at $BACKUP_BASE"
+        exit 0
+    fi
+
+    for dir in "$BACKUP_BASE"/*; do
+        if [ -d "$dir" ] && [ -f "$dir/manifest.json" ]; then
+            local NAME=$(basename "$dir")
+            local CREATED=$(python3 -c "import json; print(json.load(open('$dir/manifest.json'))['created'])" 2>/dev/null || echo "unknown")
+            local SIZE=$(du -sh "$dir" | cut -f1)
+            echo "  $NAME  ($SIZE, created: $CREATED)"
+        fi
+    done
+
+    echo ""
+    echo "To verify: $0 verify <backup_directory>"
+    echo "To restore: $0 restore <backup_directory>"
+}
+
+# =============================================================================
+# MAIN
+# =============================================================================
+case "$1" in
+    backup)
+        do_backup
+        ;;
+    restore)
+        do_restore "$2"
+        ;;
+    verify)
+        do_verify "$2"
+        ;;
+    list)
+        do_list
+        ;;
+    *)
+        echo "Usage: $0 {backup|restore|verify|list}"
+        echo ""
+        echo "Commands:"
+        echo "  backup           Create full backup of Docker volumes and git state"
+        echo "  restore <dir>    Restore from specified backup directory"
+        echo "  verify <dir>     Verify backup integrity"
+        echo "  list             List available backups"
+        exit 1
+        ;;
+esac