@ngocsangairvds/vsaf 3.2.16 → 4.0.0

package/tools/vds-scripts/memory_orchestrator/src/vds_memory_orchestrator/service/services/memory_deduplication.py

@@ -1,1761 +0,0 @@
-"""
-VDS AI Memory Deduplication Service
-
-Handles memory deduplication using vector similarity, content analysis,
-and intelligent duplicate detection with multiple algorithms and resolution strategies.
-"""
-
-import logging
-import statistics
-import uuid
-from collections import Counter
-from datetime import UTC, datetime, timedelta
-from pathlib import Path
-from typing import Any
-
-import redis
-from sqlalchemy import delete, func, select, text, update
-from sqlalchemy.ext.asyncio import AsyncSession
-from sqlalchemy.orm import joinedload
-
-try:
-    from sklearn.feature_extraction.text import TfidfVectorizer
-    from sklearn.metrics.pairwise import cosine_similarity, euclidean_distances, manhattan_distances
-except ImportError:  # pragma: no cover - optional dependency
-    TfidfVectorizer = None
-    cosine_similarity = euclidean_distances = manhattan_distances = None
-try:
-    import spacy
-except ImportError:  # pragma: no cover
-    spacy = None
-
-from vds_memory_orchestrator.service.core.config import get_config
-from vds_memory_orchestrator.service.models.ai_memory_models import MemoryStatus
-from vds_memory_orchestrator.service.models.database import (
-    MemoryDedupResolution,
-    MemoryDuplicateGroup,
-    MemoryDuplicateMember,
-    MemoryRecord,
-    MemoryRecordsTombstone,
-    MemoryVersionRecord,
-)
-from vds_memory_orchestrator.service.models.deduplication_models import (
-    AutoResolveResult,
-    BulkDeduplicationRequest,
-    BulkDeduplicationResult,
-    DeduplicationInsight,
-    DeduplicationStats,
-    DuplicateDetectionRequest,
-    DuplicateDetectionResult,
-    DuplicateGroup,
-    DuplicateStatus,
-    DuplicationAction,
-    MemoryMergeResult,
-    MergeStrategy,
-    SimilarityAlgorithm,
-    SimilarityExplanation,
-    SimilarityResult,
-)
-
-logger = logging.getLogger(__name__)
-settings = get_config()
-
-# Initialize spaCy for text processing (offline friendly)
-if spacy is not None:
-    try:
-        nlp = spacy.load("en_core_web_sm")
-        logger.info("✅ spaCy English model loaded successfully")
-    except Exception as exc:  # pragma: no cover
-        logger.warning(
-            "spaCy model unavailable; falling back to heuristic (no auto-download).",
-            exc_info=exc,
-        )
-        nlp = None
-else:
-    logger.warning("spaCy not installed; semantic similarity will use basic heuristic.")
-    nlp = None
-
-# Redis client for caching
-redis_client = redis.from_url(settings.redis_url)
-
-IMPORTANCE_WEIGHTS = {
-    "critical": 4,
-    "high": 3,
-    "medium": 2,
-    "low": 1,
-    "info": 0,
-}
-
-
-def _clamp01(value: float) -> float:
-    """Clamp a float to [0.0, 1.0]; absorbs tiny floating-point overshoot."""
-    if value is None:
-        return 0.0
-    if value < 0.0:
-        return 0.0
-    if value > 1.0:
-        return 1.0
-    return float(value)
-
-
-class MemoryDeduplicationService:
-    """Service for memory deduplication using various similarity algorithms."""
-
-    def __init__(self, db: AsyncSession):
-        self.db = db
-        self.settings = get_config()
-
-    async def detect_duplicates(
-        self,
-        request: DuplicateDetectionRequest,
-        current_user_id: uuid.UUID | None = None,
-    ) -> DuplicateDetectionResult:
-        """Detect duplicates based on the request parameters."""
-
-        start_time = datetime.now(UTC)
-
-        # Build base select
-        stmt = select(MemoryRecord)
-
-        if request.user_id:
-            stmt = stmt.where(MemoryRecord.user_id == str(request.user_id))
-        elif current_user_id and not request.check_cross_user:
-            stmt = stmt.where(MemoryRecord.user_id == str(current_user_id))
-
-        if request.time_range_days:
-            cutoff_date = datetime.now(UTC) - timedelta(days=request.time_range_days)
-            stmt = stmt.where(MemoryRecord.created_at >= cutoff_date)
-
-        if request.filters:
-            # Apply additional filters
-            if "category" in request.filters:
-                stmt = stmt.where(MemoryRecord.category == request.filters["category"])
-            if "importance" in request.filters:
-                stmt = stmt.where(MemoryRecord.importance == request.filters["importance"])
-
-        # Get memories for comparison
-        if request.memory_id:
-            # Compare specific memory with others
-            target_stmt = stmt.where(MemoryRecord.memory_id == str(request.memory_id))
-            target_memory = (await self.db.execute(target_stmt)).scalars().first()
-            if not target_memory:
-                raise ValueError(f"Memory {request.memory_id} not found")
-
-            comp_stmt = stmt.where(MemoryRecord.memory_id != str(request.memory_id)).limit(
-                request.max_results
-            )
-            comparison_memories = (await self.db.execute(comp_stmt)).scalars().all()
-            memories_to_compare = [target_memory] + list(comparison_memories)
-        else:
-            # Compare all memories within the criteria
-            all_stmt = stmt.limit(request.max_results)
-            memories_to_compare = list((await self.db.execute(all_stmt)).scalars().all())
-
-        if len(memories_to_compare) < 2:
-            return DuplicateDetectionResult(
-                duplicate_groups=[],
-                total_comparisons=0,
-                processing_time_ms=0,
-                algorithm_used=request.algorithm,
-                threshold_used=request.similarity_threshold,
-                auto_resolvable_count=0,
-                metadata={"message": "Insufficient memories for comparison"},
-            )
-
-        # Calculate similarities
-        similarities = []
-        total_comparisons = max(len(memories_to_compare) * (len(memories_to_compare) - 1) // 2, 0)
-
-        if request.algorithm == SimilarityAlgorithm.SEMANTIC:
-            similarities = await self._calculate_semantic_similarities(
-                memories_to_compare, request.similarity_threshold, request
-            )
-        elif request.algorithm == SimilarityAlgorithm.COSINE:
-            similarities = await self._calculate_cosine_similarities(
-                memories_to_compare, request.similarity_threshold, request
-            )
-        elif request.algorithm == SimilarityAlgorithm.EUCLIDEAN:
-            similarities = await self._calculate_euclidean_similarities(
-                memories_to_compare, request.similarity_threshold, request
-            )
-        elif request.algorithm == SimilarityAlgorithm.MANHATTAN:
-            similarities = await self._calculate_manhattan_similarities(
-                memories_to_compare, request.similarity_threshold, request
-            )
-        elif request.algorithm == SimilarityAlgorithm.JACCARD:
-            similarities = await self._calculate_jaccard_similarities(
-                memories_to_compare, request.similarity_threshold, request
-            )
-
-        # Pre-compute score map so _group_similar_memories can stay sync
-        score_map: dict[str, float] = {
-            m.memory_id: self._compute_memory_score(m) for m in memories_to_compare
-        }
-
-        # Group similar memories
-        duplicate_groups = self._group_similar_memories(
-            similarities,
-            request.similarity_threshold,
-            request.algorithm,
-            score_map=score_map,
-        )
-
-        await self._persist_duplicate_groups(duplicate_groups)
-
-        # Calculate processing time
-        processing_time = (datetime.now(UTC) - start_time).total_seconds() * 1000
-
-        # Check for auto-resolvable groups
-        auto_resolvable_count = sum(
-            1 for group in duplicate_groups if self._is_auto_resolvable(group)
-        )
-
-        for group in duplicate_groups:
-            metadata = group.metadata or {}
-            metadata.setdefault("processing_time_ms", int(processing_time))
-            metadata.setdefault("algorithm", request.algorithm.value)
-            group.metadata = metadata
-
-        return DuplicateDetectionResult(
-            duplicate_groups=duplicate_groups,
-            total_comparisons=total_comparisons,
-            processing_time_ms=int(processing_time),
-            algorithm_used=request.algorithm,
-            threshold_used=request.similarity_threshold,
-            auto_resolvable_count=auto_resolvable_count,
-            recommendations=self._generate_resolution_recommendations(duplicate_groups),
-            metadata={
-                "memories_compared": len(memories_to_compare),
-                "user_id": str(request.user_id) if request.user_id else str(current_user_id),
-                "timestamp": datetime.now(UTC).isoformat(),
-            },
-        )
-
-    async def calculate_similarity(
-        self,
-        memory1_id: uuid.UUID,
-        memory2_id: uuid.UUID,
-        algorithm: SimilarityAlgorithm = SimilarityAlgorithm.COSINE,
-        include_content: bool = True,
-        include_metadata: bool = True,
-    ) -> SimilarityResult:
-        """Calculate similarity between two specific memories."""
-
-        # Get memories
-        memory1 = (
-            (
-                await self.db.execute(
-                    select(MemoryRecord).where(MemoryRecord.memory_id == str(memory1_id))
-                )
-            )
-            .scalars()
-            .first()
-        )
-        memory2 = (
-            (
-                await self.db.execute(
-                    select(MemoryRecord).where(MemoryRecord.memory_id == str(memory2_id))
-                )
-            )
-            .scalars()
-            .first()
-        )
-
-        if not memory1 or not memory2:
-            raise ValueError("One or both memories not found")
-
-        # Calculate individual similarities
-        content_similarity = 0.0
-        title_similarity = 0.0
-        metadata_similarity = 0.0
-
-        if include_content and memory1.content and memory2.content:
-            content_similarity = await self._text_similarity(memory1.content, memory2.content)
-
-        if memory1.title and memory2.title:
-            title_similarity = await self._text_similarity(memory1.title, memory2.title)
-
-        if include_metadata:
-            metadata_similarity = await self._metadata_similarity(memory1, memory2)
-
-        # Calculate weighted overall similarity
-        weights = {"content": 0.7, "title": 0.2, "metadata": 0.1}
-
-        overall_similarity = (
-            (content_similarity * weights["content"] if include_content else 0)
-            + (title_similarity * weights["title"] if memory1.title and memory2.title else 0)
-            + (metadata_similarity * weights["metadata"] if include_metadata else 0)
-        )
-
-        # Normalize weights if some components are missing
-        weight_sum = (
-            (weights["content"] if include_content else 0)
-            + (weights["title"] if memory1.title and memory2.title else 0)
-            + (weights["metadata"] if include_metadata else 0)
-        )
-
-        if weight_sum > 0:
-            overall_similarity /= weight_sum
-        overall_similarity = _clamp01(overall_similarity)
-        content_similarity = _clamp01(content_similarity)
-        title_similarity = _clamp01(title_similarity)
-        metadata_similarity = _clamp01(metadata_similarity)
-
-        # Calculate confidence based on data completeness
-        completeness_factor = 0.0
-        if include_content and memory1.content and memory2.content:
-            completeness_factor += 0.5
-        if memory1.title and memory2.title:
-            completeness_factor += 0.3
-        if include_metadata:
-            completeness_factor += 0.2
-
-        confidence = min(completeness_factor, 1.0)
-
-        return SimilarityResult(
-            memory_id_1=str(memory1_id),
-            memory_id_2=str(memory2_id),
-            overall_similarity=_clamp01(overall_similarity),
-            content_similarity=_clamp01(content_similarity) if include_content else None,
-            title_similarity=_clamp01(title_similarity)
-            if memory1.title and memory2.title
-            else None,
-            metadata_similarity=_clamp01(metadata_similarity) if include_metadata else None,
-            algorithm=algorithm,
-            confidence=_clamp01(confidence),
-            comparison_timestamp=datetime.now(UTC),
-        )
-
-    async def get_similarity_explanation(
-        self,
-        memory1_id: uuid.UUID,
-        memory2_id: uuid.UUID,
-        algorithm: SimilarityAlgorithm = SimilarityAlgorithm.COSINE,
-    ) -> SimilarityExplanation:
-        """Get detailed explanation of similarity between two memories."""
-
-        # Calculate similarity first
-        similarity_result = await self.calculate_similarity(memory1_id, memory2_id, algorithm)
-
-        # Get memories for detailed analysis
-        memory1 = (
-            (
-                await self.db.execute(
-                    select(MemoryRecord).where(MemoryRecord.memory_id == str(memory1_id))
-                )
-            )
-            .scalars()
-            .first()
-        )
-        memory2 = (
-            (
-                await self.db.execute(
-                    select(MemoryRecord).where(MemoryRecord.memory_id == str(memory2_id))
-                )
-            )
-            .scalars()
-            .first()
-        )
-
-        # Analyze key factors
-        key_factors = []
-        shared_elements = []
-        differences = []
-
-        if memory1.content and memory2.content:
-            # Content analysis
-            words1 = set(memory1.content.lower().split())
-            words2 = set(memory2.content.lower().split())
-
-            common_words = words1.intersection(words2)
-            shared_elements.extend(list(common_words)[:10])  # Top 10 common words
-
-            unique_words1 = words1 - words2
-            unique_words2 = words2 - words1
-            differences.extend([f"Memory 1 unique: {', '.join(list(unique_words1)[:5])}"])
-            differences.extend([f"Memory 2 unique: {', '.join(list(unique_words2)[:5])}"])
-
-            if len(common_words) / max(len(words1), len(words2)) > 0.3:
-                key_factors.append("High content overlap")
-
-        if memory1.title and memory2.title:
-            # Title analysis
-            if memory1.title.lower() == memory2.title.lower():
-                key_factors.append("Identical titles")
-            elif (
-                memory1.title.lower() in memory2.title.lower()
-                or memory2.title.lower() in memory1.title.lower()
-            ):
-                key_factors.append("Similar titles")
-
-        # Metadata analysis
-        metadata_similarity = await self._metadata_similarity(memory1, memory2)
-        if metadata_similarity > 0.8:
-            key_factors.append("Similar metadata patterns")
-
-        # Generate confidence explanation
-        confidence_explanation = "Confidence based on data completeness: "
-        if similarity_result.content_similarity is not None:
-            confidence_explanation += "Content available, "
-        if similarity_result.title_similarity is not None:
-            confidence_explanation += "Title available, "
-        confidence_explanation += f"Overall confidence: {similarity_result.confidence:.2f}"
-
-        return SimilarityExplanation(
-            overall_score=similarity_result.overall_similarity,
-            content_similarity=similarity_result.content_similarity or 0.0,
-            title_similarity=similarity_result.title_similarity or 0.0,
-            metadata_similarity=similarity_result.metadata_similarity or 0.0,
-            key_factors=key_factors,
-            shared_elements=shared_elements,
-            differences=differences,
-            confidence_explanation=confidence_explanation,
-            algorithm_details={
-                "algorithm": algorithm.value,
-                "weights": {"content": 0.7, "title": 0.2, "metadata": 0.1},
-            },
-        )
-
-    async def resolve_duplicates(
-        self, group_id: uuid.UUID, merge_strategy: MergeStrategy, resolved_by: uuid.UUID
-    ) -> MemoryMergeResult:
-        """Resolve duplicates in a group using the specified strategy."""
-
-        start_time = datetime.now(UTC)
-
-        # Find the duplicate group
-        # In a real implementation, you would have a DuplicateGroup table
-        # For now, we'll simulate finding and resolving duplicates
-
-        # Get memories in the group (simplified - would be stored in database)
-        memory_ids = await self._get_group_memory_ids(group_id)
-
-        if not memory_ids:
-            raise ValueError(f"No memories found for group {group_id}")
-
-        # Get actual memory objects
-        str_ids = [str(mid) for mid in memory_ids]
-        memories = list(
-            (await self.db.execute(select(MemoryRecord).where(MemoryRecord.memory_id.in_(str_ids))))
-            .scalars()
-            .all()
-        )
-
-        if len(memories) < 2:
-            raise ValueError("Need at least 2 memories to resolve duplicates")
-
-        # Apply merge strategy
-        if merge_strategy.action == DuplicationAction.MERGE:
-            merge_result = await self._merge_memories(memories, merge_strategy)
-        elif merge_strategy.action == DuplicationAction.DELETE:
-            merge_result = await self._delete_duplicates(memories, merge_strategy)
-        elif merge_strategy.action == DuplicationAction.ARCHIVE:
-            merge_result = await self._archive_duplicates(memories, merge_strategy)
-        elif merge_strategy.action == DuplicationAction.FLAG:
-            merge_result = await self._flag_duplicates(memories, merge_strategy)
-        else:
-            raise ValueError(f"Unsupported action: {merge_strategy.action}")
-
-        # Record resolution
-        await self._record_resolution(group_id, merge_strategy.action, resolved_by, merge_result)
-
-        await self.db.commit()
-
-        processing_time = (datetime.now(UTC) - start_time).total_seconds() * 1000
-
-        return MemoryMergeResult(
-            success=merge_result["success"],
-            primary_memory_id=merge_result.get("primary_memory_id"),
-            merged_memory_ids=merge_result.get("merged_memory_ids", []),
-            created_version_ids=merge_result.get("created_version_ids", []),
-            conflicts=merge_result.get("conflicts", []),
-            warnings=merge_result.get("warnings", []),
-            merge_summary=merge_result.get("summary", {}),
-            processing_time_ms=int(processing_time),
-        )
-
-    async def get_deduplication_stats(
-        self, user_id: uuid.UUID | None = None, days: int = 30
-    ) -> DeduplicationStats:
-        """Get deduplication statistics for a user or globally."""
-        base_stmt = select(MemoryDuplicateGroup)
-
-        if user_id:
-            group_ids_stmt = (
-                select(MemoryDuplicateGroup.id)
-                .join(
-                    MemoryDuplicateMember,
-                    MemoryDuplicateMember.group_id == MemoryDuplicateGroup.id,
-                )
-                .join(
-                    MemoryRecord,
-                    MemoryRecord.memory_id == MemoryDuplicateMember.memory_id,
-                )
-                .where(MemoryRecord.user_id == str(user_id))
-                .distinct()
-            )
-            group_ids = list((await self.db.execute(group_ids_stmt)).scalars().all())
-            base_stmt = (
-                base_stmt.where(MemoryDuplicateGroup.id.in_(group_ids))
-                if group_ids
-                else base_stmt.where(text("1=0"))
-            )
-
-        groups_stmt = base_stmt.options(joinedload(MemoryDuplicateGroup.members))
-        groups = list((await self.db.execute(groups_stmt)).scalars().unique().all())
-        total_groups = len(groups)
-        potential_duplicates = sum(len(group.members) for group in groups)
-        resolved_groups = sum(
-            1 for group in groups if group.status == DuplicateStatus.RESOLVED.value
-        )
-        auto_resolved = sum(
-            1 for group in groups if (group.group_metadata or {}).get("auto_resolvable")
-        )
-        average_similarity = (
-            statistics.fmean([group.similarity_score for group in groups]) if groups else 0.0
-        )
-        processing_time_total_ms = sum(
-            (group.group_metadata or {}).get("processing_time_ms", 0) for group in groups
-        )
-
-        memory_count_stmt = select(func.count(MemoryRecord.memory_id))
-        if user_id:
-            memory_count_stmt = memory_count_stmt.where(MemoryRecord.user_id == str(user_id))
-        total_memories = (await self.db.execute(memory_count_stmt)).scalar_one()
-
-        return DeduplicationStats(
-            total_memories=total_memories,
-            duplicate_groups_found=total_groups,
-            potential_duplicates=potential_duplicates,
-            resolved_duplicates=resolved_groups,
-            auto_resolved=auto_resolved,
-            manual_review_required=max(resolved_groups - auto_resolved, 0),
-            average_similarity_score=average_similarity,
-            processing_time_total_ms=int(processing_time_total_ms),
-            time_period_days=days,
-            algorithm=SimilarityAlgorithm.COSINE,
-            last_analysis_date=datetime.now(UTC),
-        )
-
-    async def schedule_bulk_deduplication(
-        self, request: BulkDeduplicationRequest
-    ) -> BulkDeduplicationResult:
-        """Schedule and execute bulk deduplication operation."""
-
-        operation_id = uuid.uuid4()
-        start_time = datetime.now(UTC)
-
-        try:
-            # Get memories to process
-            stmt = select(MemoryRecord)
-
-            if request.user_ids:
-                user_ids = [str(uid) for uid in request.user_ids]
-                stmt = stmt.where(MemoryRecord.user_id.in_(user_ids))
-
-            cutoff_date = datetime.now(UTC) - timedelta(days=request.time_range_days)
-            stmt = stmt.where(MemoryRecord.created_at >= cutoff_date)
-
-            # Batch processing for large datasets
-            all_memories = []
-            offset = 0
-
-            while offset < request.max_batch_size:
-                batch = list(
-                    (await self.db.execute(stmt.offset(offset).limit(1000))).scalars().all()
-                )
-                if not batch:
-                    break
-                all_memories.extend(batch)
-                offset += 1000
-
-            if len(all_memories) < 2:
-                return BulkDeduplicationResult(
-                    operation_id=operation_id,
-                    status="completed",
-                    total_memories_processed=len(all_memories),
-                    duplicate_groups_found=0,
-                    duplicates_resolved=0,
-                    warnings=["Insufficient memories for deduplication"],
-                    processing_time_ms=0,
-                    started_at=start_time,
-                    completed_at=datetime.now(UTC),
-                )
-
-            # Process duplicates in batches
-            duplicate_groups = []
-            duplicates_resolved = 0
-            errors = []
-            warnings = []
-
-            batch_size = 100
-            for i in range(0, len(all_memories), batch_size):
-                batch_memories = all_memories[i : i + batch_size]
-
-                try:
-                    # Detect duplicates for this batch
-                    detection_request = DuplicateDetectionRequest(
-                        similarity_threshold=request.similarity_threshold,
-                        max_results=len(batch_memories),
-                        algorithm=SimilarityAlgorithm.COSINE,  # Use efficient algorithm for bulk
-                        include_content=True,
-                        include_metadata=True,
-                        check_cross_user=request.user_ids is None,
-                    )
-
-                    batch_result = await self.detect_duplicates(detection_request)
-                    duplicate_groups.extend(batch_result.duplicate_groups)
-
-                    # Auto-resolve if requested
-                    if request.auto_resolve and not request.dry_run:
-                        for group in batch_result.duplicate_groups:
-                            if self._is_auto_resolvable(group):
-                                try:
-                                    merge_strategy = MergeStrategy(
-                                        action=DuplicationAction.MERGE,
-                                        auto_resolve=True,
-                                    )
-                                    await self.resolve_duplicates(
-                                        group.group_id,
-                                        merge_strategy,
-                                        self._ensure_uuid("system"),
-                                    )
-                                    duplicates_resolved += len(group.memory_ids) - 1
-                                except Exception as e:
-                                    errors.append(
-                                        f"Failed to resolve group {group.group_id}: {str(e)}"
-                                    )
-
-                except Exception as e:
-                    errors.append(f"Error processing batch {i // batch_size + 1}: {str(e)}")
-
-            # Calculate statistics
-            processing_time = (datetime.now(UTC) - start_time).total_seconds() * 1000
-
-            return BulkDeduplicationResult(
-                operation_id=operation_id,
-                status="completed",
-                total_memories_processed=len(all_memories),
-                duplicate_groups_found=len(duplicate_groups),
-                duplicates_resolved=duplicates_resolved,
-                errors=errors,
-                warnings=warnings,
-                processing_time_ms=int(processing_time),
-                memory_saved_mb=await self._calculate_memory_savings(duplicate_groups),
-                started_at=start_time,
-                completed_at=datetime.now(UTC),
-            )
-
-        except Exception as e:
-            return BulkDeduplicationResult(
-                operation_id=operation_id,
-                status="failed",
-                total_memories_processed=0,
-                duplicate_groups_found=0,
-                duplicates_resolved=0,
-                errors=[str(e)],
-                processing_time_ms=0,
-                started_at=start_time,
-                completed_at=datetime.now(UTC),
-            )
-
-    async def generate_deduplication_insights(
-        self, user_id: uuid.UUID | None = None, days: int = 30
-    ) -> DeduplicationInsight:
-        """Generate insights from deduplication analysis."""
-
-        stats = await self.get_deduplication_stats(user_id, days)
-
-        group_stmt = select(MemoryDuplicateGroup).options(joinedload(MemoryDuplicateGroup.members))
-        if user_id:
-            group_stmt = (
-                group_stmt.join(MemoryDuplicateMember)
-                .join(
-                    MemoryRecord,
-                    MemoryRecord.memory_id == MemoryDuplicateMember.memory_id,
-                )
-                .where(MemoryRecord.user_id == str(user_id))
-            )
-        groups = list((await self.db.execute(group_stmt)).scalars().unique().all())
-
-        member_ids = {member.memory_id for group in groups for member in group.members}
-        memory_map = await self._load_memories_by_ids(member_ids)
-
-        category_counter = Counter(
-            memory_map[mem_id].category
-            for mem_id in member_ids
-            if mem_id in memory_map and memory_map[mem_id].category
-        )
-        duplicate_hotspots = [
-            {
-                "category": category,
-                "duplicate_rate": round(count / max(1, stats.potential_duplicates), 3),
-                "common_factors": ["High content overlap", "Frequent edits"],
-            }
-            for category, count in category_counter.most_common(3)
-        ]
-        if not duplicate_hotspots:
-            duplicate_hotspots = [
-                {
-                    "category": "general",
-                    "duplicate_rate": 0.0,
-                    "common_factors": ["Insufficient data"],
-                }
-            ]
-
-        user_counter = Counter(
-            memory_map[mem_id].user_id
-            for mem_id in member_ids
-            if mem_id in memory_map and memory_map[mem_id].user_id
-        )
-        user_behavior_insights = [
-            {
-                "user_id": user,
-                "duplicate_frequency": "high" if count > 3 else "moderate",
-                "resolution_rate": stats.resolved_duplicates / max(1, stats.duplicate_groups_found),
-                "preferred_action": "merge" if stats.auto_resolved else "review",
-            }
-            for user, count in user_counter.most_common(3)
-        ] or [
-            {
-                "user_id": str(user_id) if user_id else "aggregate",
-                "duplicate_frequency": "low",
-                "resolution_rate": 0.0,
-                "preferred_action": "review",
-            }
-        ]
-
-        common_patterns = []
-        if stats.auto_resolved:
-            common_patterns.append("High-confidence duplicate pairs were auto-resolved")
-        if stats.potential_duplicates > stats.duplicate_groups_found:
-            common_patterns.append("Multiple duplicates share the same representative memory")
-        if not common_patterns:
-            common_patterns.append("Duplicate rate currently minimal")
-
-        improvement_suggestions = []
-        if stats.manual_review_required:
-            improvement_suggestions.append(
-                "Prioritize manual review queue to clear pending duplicate groups"
-            )
-        if stats.average_similarity_score < 0.8:
-            improvement_suggestions.append(
-                "Adjust similarity threshold or enrich embeddings to improve clustering precision"
-            )
-        if stats.auto_resolved == 0:
-            improvement_suggestions.append(
-                "Enable auto-resolution for two-memory groups exceeding 0.9 similarity"
-            )
-        if not improvement_suggestions:
-            improvement_suggestions.append(
-                "Maintain current deduplication cadence; no blocking issues detected"
-            )
-
-        precision = stats.resolved_duplicates / max(1, stats.duplicate_groups_found)
-        recall = stats.resolved_duplicates / max(1, stats.potential_duplicates)
-        quality_metrics = {
-            "precision": round(precision, 3),
-            "recall": round(recall, 3),
-            "f1_score": round(2 * precision * recall / max(precision + recall, 1e-9), 3),
-        }
-
-        reclaimed_mb = await self._calculate_memory_savings(groups)
-
-        efficiency_metrics = {
-            "processing_speed_ms_per_comparison": round(
-                stats.processing_time_total_ms / max(stats.potential_duplicates, 1), 3
-            ),
-            "memory_reduction_percent": round(
-                (reclaimed_mb / max(stats.total_memories, 1)) * 100, 2
-            ),
-            "auto_resolution_rate": round(
-                stats.auto_resolved / max(stats.duplicate_groups_found, 1), 3
-            ),
-        }
-
-        cost_savings = {
-            "storage_saved_mb": round(reclaimed_mb, 3),
-            "processing_time_saved_hours": round(
-                (stats.potential_duplicates - stats.resolved_duplicates) * 0.001, 2
-            ),
-            "user_productivity_gain_percent": round(precision * 100, 2),
-        }
-
-        return DeduplicationInsight(
-            duplicate_hotspots=duplicate_hotspots,
-            common_patterns=common_patterns,
-            user_behavior_insights=user_behavior_insights,
-            improvement_suggestions=improvement_suggestions,
-            quality_metrics=quality_metrics,
-            efficiency_metrics=efficiency_metrics,
-            cost_savings=cost_savings,
-            generated_at=datetime.now(UTC),
-            confidence_level=min(1.0, precision + 0.2),
-        )
-
-    # Private helper methods
-
-    async def _text_similarity(self, text1: str, text2: str) -> float:
-        """Calculate text similarity using TF-IDF. Clamped to [0, 1]."""
-        if TfidfVectorizer is None or cosine_similarity is None:
-            return _clamp01(self._basic_text_similarity(text1, text2))
-
-        try:
-            vectorizer = TfidfVectorizer(stop_words="english", lowercase=True)
-            tfidf_matrix = vectorizer.fit_transform([text1, text2])
-            similarity = cosine_similarity(tfidf_matrix[0:1], tfidf_matrix[1:2])[0][0]
-            return _clamp01(float(similarity))
-        except Exception as e:
-            logger.warning(f"Error calculating text similarity: {e}")
-            return _clamp01(self._basic_text_similarity(text1, text2))
-
-    def _basic_text_similarity(self, text1: str, text2: str) -> float:
-        words1 = set(text1.lower().split())
-        words2 = set(text2.lower().split())
-        if not words1 and not words2:
-            return 0.0
-        intersection = len(words1 & words2)
-        union = len(words1 | words2) or 1
-        return intersection / union
-
-    async def _metadata_similarity(self, memory1: MemoryRecord, memory2: MemoryRecord) -> float:
-        """Calculate metadata similarity between two memories."""
-        similarity_score = 0.0
-        total_factors = 0
-
-        # Compare categories
-        if hasattr(memory1, "category") and hasattr(memory2, "category"):
-            if memory1.category == memory2.category:
-                similarity_score += 1.0
-            total_factors += 1.0
-
-        # Compare importance levels
-        if memory1.importance and memory2.importance:
-            if memory1.importance == memory2.importance:
-                similarity_score += 1.0
-            total_factors += 1.0
-
-        # Compare tags (if available)
-        if hasattr(memory1, "tags") and hasattr(memory2, "tags"):
-            tags1 = set(memory1.tags or [])
-            tags2 = set(memory2.tags or [])
-            if tags1 or tags2:
-                intersection = tags1.intersection(tags2)
-                union = tags1.union(tags2)
-                jaccard_similarity = len(intersection) / len(union) if union else 0
-                similarity_score += jaccard_similarity
-                total_factors += 1.0
-
-        return _clamp01(similarity_score / total_factors if total_factors > 0 else 0.0)
-
-    async def _calculate_cosine_similarities(
-        self,
-        memories: list[MemoryRecord],
-        threshold: float,
-        request: DuplicateDetectionRequest,
-    ) -> list[SimilarityResult]:
-        """Calculate cosine similarities between all memory pairs."""
-        similarities = []
-
-        for i in range(len(memories)):
-            for j in range(i + 1, len(memories)):
-                similarity = await self.calculate_similarity(
-                    memories[i].memory_id,
-                    memories[j].memory_id,
-                    request.algorithm,
-                    request.include_content,
-                    request.include_metadata,
-                )
-
-                if similarity.overall_similarity >= threshold:
-                    similarities.append(similarity)
-
-        return similarities
-
-    async def _calculate_semantic_similarities(
-        self,
-        memories: list[MemoryRecord],
-        threshold: float,
-        request: DuplicateDetectionRequest,
-    ) -> list[SimilarityResult]:
-        """Calculate semantic similarities using embeddings."""
-
-        # In a real implementation, you would use actual embeddings
-        # For now, we'll fall back to TF-IDF as semantic approximation
-        return await self._calculate_cosine_similarities(memories, threshold, request)
-
-    async def _calculate_euclidean_similarities(
-        self,
-        memories: list[MemoryRecord],
-        threshold: float,
-        request: DuplicateDetectionRequest,
-    ) -> list[SimilarityResult]:
-        """Calculate Euclidean distance similarities."""
-        similarities = []
-
-        for i in range(len(memories)):
-            for j in range(i + 1, len(memories)):
-                # Calculate similarity components
-                content_sim = 0.0
-                title_sim = 0.0
-                metadata_sim = 0.0
-
-                if request.include_content and memories[i].content and memories[j].content:
-                    vectorizer = TfidfVectorizer(stop_words="english", lowercase=True)
-                    tfidf_matrix = vectorizer.fit_transform(
-                        [memories[i].content, memories[j].content]
-                    )
-                    distance = euclidean_distances(tfidf_matrix[0:1], tfidf_matrix[1:2])[0][0]
-                    # Convert distance to similarity (inverse relationship)
-                    content_sim = max(0, 1 - distance / 10)  # Normalize to 0-1 range
-
-                if memories[i].title and memories[j].title:
-                    title_sim = await self._text_similarity(memories[i].title, memories[j].title)
-
-                if request.include_metadata:
-                    metadata_sim = await self._metadata_similarity(memories[i], memories[j])
-
-                # Weighted combination
-                weights = {"content": 0.7, "title": 0.2, "metadata": 0.1}
-                overall_sim = (
-                    content_sim * weights["content"]
-                    + title_sim * weights["title"]
-                    + metadata_sim * weights["metadata"]
-                )
-
-                if overall_sim >= threshold:
-                    similarity = SimilarityResult(
-                        memory_id_1=memories[i].memory_id,
-                        memory_id_2=memories[j].memory_id,
-                        overall_similarity=_clamp01(overall_sim),
-                        content_similarity=_clamp01(content_sim)
-                        if request.include_content
-                        else None,
-                        title_similarity=_clamp01(title_sim)
-                        if memories[i].title and memories[j].title
-                        else None,
-                        metadata_similarity=_clamp01(metadata_sim)
-                        if request.include_metadata
-                        else None,
-                        algorithm=request.algorithm,
-                        confidence=0.8,
-                        comparison_timestamp=datetime.now(UTC),
-                    )
-                    similarities.append(similarity)
-
-        return similarities
-
-    async def _calculate_manhattan_similarities(
-        self,
-        memories: list[MemoryRecord],
-        threshold: float,
-        request: DuplicateDetectionRequest,
-    ) -> list[SimilarityResult]:
-        """Calculate Manhattan distance similarities."""
-        # Similar to Euclidean but using Manhattan distance
-        return await self._calculate_euclidean_similarities(memories, threshold, request)
-
-    async def _calculate_jaccard_similarities(
-        self,
-        memories: list[MemoryRecord],
-        threshold: float,
-        request: DuplicateDetectionRequest,
-    ) -> list[SimilarityResult]:
-        """Calculate Jaccard similarities."""
-        similarities = []
-
-        for i in range(len(memories)):
-            for j in range(i + 1, len(memories)):
-                # Get text tokens
-                tokens1 = set()
-                tokens2 = set()
-
-                if request.include_content:
-                    content1 = memories[i].content.lower().split() if memories[i].content else []
-                    content2 = memories[j].content.lower().split() if memories[j].content else []
-                    tokens1.update(content1)
-                    tokens2.update(content2)
-
-                if memories[i].title:
-                    tokens1.update(memories[i].title.lower().split())
-                if memories[j].title:
-                    tokens2.update(memories[j].title.lower().split())
-
-                # Calculate Jaccard similarity
-                intersection = tokens1.intersection(tokens2)
-                union = tokens1.union(tokens2)
-                jaccard_sim = len(intersection) / len(union) if union else 0
-
-                if jaccard_sim >= threshold:
-                    jaccard_sim_c = _clamp01(jaccard_sim)
-                    similarity = SimilarityResult(
-                        memory_id_1=memories[i].memory_id,
-                        memory_id_2=memories[j].memory_id,
-                        overall_similarity=jaccard_sim_c,
-                        content_similarity=jaccard_sim_c if request.include_content else None,
-                        title_similarity=jaccard_sim_c
-                        if memories[i].title and memories[j].title
-                        else None,
-                        metadata_similarity=None,
-                        algorithm=request.algorithm,
-                        confidence=0.7,
-                        comparison_timestamp=datetime.now(UTC),
-                    )
-                    similarities.append(similarity)
-
-        return similarities
-
-    def _group_similar_memories(
-        self,
-        similarities: list[SimilarityResult],
-        threshold: float,
-        algorithm: SimilarityAlgorithm,
-        score_map: dict[str, float] | None = None,
-    ) -> list[DuplicateGroup]:
-        """Group similar memories into duplicate groups."""
-        groups = []
-        processed_ids = set()
-        scores = score_map or {}
-
-        for similarity in similarities:
-            if similarity.memory_id_1 in processed_ids or similarity.memory_id_2 in processed_ids:
-                continue
-
-            # Find all related memories through similarity chains
-            group_ids = {similarity.memory_id_1, similarity.memory_id_2}
-
-            # Find connected components in similarity graph
-            changed = True
-            while changed:
-                changed = False
-                for sim in similarities:
-                    if (sim.memory_id_1 in group_ids) != (sim.memory_id_2 in group_ids):
-                        if sim.memory_id_1 in group_ids:
-                            group_ids.add(sim.memory_id_2)
-                        else:
-                            group_ids.add(sim.memory_id_1)
-                        changed = True
-
-            # Create group
-            if len(group_ids) > 1:
-                representative_id = max(group_ids, key=lambda mid: scores.get(mid, 0.0))
-                auto_resolvable = self._score_auto_resolution(
-                    group_ids, similarity.overall_similarity
-                )
-
-                group = DuplicateGroup(
-                    similarity_score=similarity.overall_similarity,
-                    memory_ids=list(group_ids),
-                    representative_memory_id=representative_id,
-                    status=DuplicateStatus.PENDING,
-                    auto_resolvable=auto_resolvable,
-                    metadata={
-                        "algorithm": algorithm.value,
-                        "threshold": threshold,
-                        "comparisons": len(group_ids) - 1,
-                    },
-                )
-                groups.append(group)
-                processed_ids.update(group_ids)
-
-        return groups
-
-    def _score_auto_resolution(self, memory_ids: set[str], similarity: float) -> bool:
-        return similarity >= 0.9 and len(memory_ids) <= 2
-
-    def _compute_memory_score(self, record: MemoryRecord) -> float:
-        """Score a memory record using importance and recency (pure function, no DB)."""
-        if record is None:
-            return 0.0
-        importance_raw = (record.importance or "").lower()
-        importance_score = IMPORTANCE_WEIGHTS.get(importance_raw, 1)
-        timestamp = (
-            record.updated_at or record.created_at or datetime.min.replace(tzinfo=UTC)
-        ).timestamp()
-        return importance_score * 1_000_000_000 + timestamp
-
-    def _is_auto_resolvable(self, group: DuplicateGroup) -> bool:
-        """Check if a duplicate group can be automatically resolved."""
-        return group.auto_resolvable
-
-    def _generate_resolution_recommendations(
-        self, groups: list[DuplicateGroup]
-    ) -> list[dict[str, Any]]:
-        """Generate AI-powered resolution recommendations."""
-        recommendations = []
-
-        for group in groups:
-            recommendation = {
-                "group_id": str(group.group_id),
-                "similarity_score": group.similarity_score,
-                "recommended_action": "merge" if group.auto_resolvable else "review",
-                "confidence": min(
-                    group.similarity_score + (0.1 if group.auto_resolvable else 0.0),
-                    1.0,
-                ),
-                "reasoning": (
-                    "Auto-resolution criteria met (similarity >= 0.9, group size <= 2)"
-                    if group.auto_resolvable
-                    else (
-                        f"Auto-resolution declined: "
-                        f"similarity={group.similarity_score:.2f} (need >= 0.9), "
-                        f"group_size={len(group.memory_ids)} (need <= 2)"
-                    )
-                ),
-            }
-            recommendations.append(recommendation)
-
-        return recommendations
-
-    def _select_primary_memory(
-        self,
-        memories: list[MemoryRecord],
-        strategy: MergeStrategy,
-    ) -> MemoryRecord:
-        if strategy.keep_memory_id:
-            keep_id = str(strategy.keep_memory_id)
-            for memory in memories:
-                if memory.memory_id == keep_id:
-                    return memory
-
-        def score(memory: MemoryRecord) -> tuple[int, datetime]:
-            importance_value = IMPORTANCE_WEIGHTS.get(memory.importance.lower(), 1)
-            created_at = memory.created_at or datetime.min.replace(tzinfo=UTC)
-            return importance_value, created_at
-
-        return max(memories, key=score)
-
-    async def _create_version_snapshot(
-        self,
-        memory: MemoryRecord,
-        reason: str,
-        created_by: str | None,
-    ) -> MemoryVersionRecord:
-        next_version = (memory.total_versions or 0) + 1
-        version = MemoryVersionRecord(
-            memory_id=memory.memory_id,
-            version_number=next_version,
-            content=memory.content,
-            category=memory.category,
-            importance=memory.importance,
-            metadata=memory.metadata_dict,
-            is_active=False,
-            created_at=datetime.now(UTC),
-            created_by=created_by,
-            change_reason=reason,
-            content_size_bytes=len(memory.content.encode("utf-8")) if memory.content else 0,
-            processing_time_ms=0.0,
-        )
-        self.db.add(version)
-        await self.db.flush([version])
-        memory.total_versions = next_version
-        memory.last_version_created = version.created_at
-        return version
-
-    def _merge_metadata(self, base: dict[str, Any], incoming: dict[str, Any]) -> dict[str, Any]:
-        merged = dict(base or {})
-        for key, value in (incoming or {}).items():
-            if key not in merged:
-                merged[key] = value
-                continue
-            if isinstance(merged[key], dict) and isinstance(value, dict):
-                merged[key] = self._merge_metadata(merged[key], value)
-            elif isinstance(merged[key], list) and isinstance(value, list):
-                merged[key] = list(dict.fromkeys([*merged[key], *value]))
-            else:
-                merged[key] = value
-        return merged
-
-    def _ensure_uuid(self, value: str | uuid.UUID | None) -> uuid.UUID:
-        if isinstance(value, uuid.UUID):
-            return value
-        if value is None:
-            return uuid.uuid4()
-        try:
-            return uuid.UUID(str(value))
-        except ValueError:
-            return uuid.uuid5(uuid.NAMESPACE_URL, str(value))
-
-    async def _merge_memories(
-        self, memories: list[MemoryRecord], strategy: MergeStrategy
-    ) -> dict[str, Any]:
-        """Merge duplicate memories into a single canonical memory."""
-        primary = self._select_primary_memory(memories, strategy)
-        duplicates = [m for m in memories if m.memory_id != primary.memory_id]
-        actor = str(strategy.keep_memory_id) if strategy.keep_memory_id else primary.user_id
-
-        created_versions = [
-            await self._create_version_snapshot(primary, "Pre-merge snapshot", actor)
-        ]
-
-        if strategy.merge_content:
-            content_segments = [primary.content] + [m.content for m in duplicates if m.content]
-            primary.content = strategy.content_separator.join(
-                segment for segment in content_segments if segment
-            )
-
-        if strategy.merge_metadata:
-            merged_metadata = primary.metadata_dict
-            for memory in duplicates:
-                merged_metadata = self._merge_metadata(merged_metadata, memory.metadata_dict)
-            primary.metadata_dict = merged_metadata
-
-        if strategy.merge_tags:
-            tags = set(primary.metadata_dict.get("tags", []))
-            for memory in duplicates:
-                tags.update(memory.metadata_dict.get("tags", []))
-            if tags:
-                metadata = primary.metadata_dict
-                metadata["tags"] = sorted(tags)
-                primary.metadata_dict = metadata
-
-        primary.updated_at = datetime.now(UTC)
-
-        for memory in duplicates:
-            snapshot = await self._create_version_snapshot(
-                memory, f"Merged into {primary.memory_id}", actor
-            )
-            created_versions.append(snapshot)
-            memory.status = MemoryStatus.ARCHIVED.value
-            memory.metadata_dict = self._merge_metadata(
-                memory.metadata_dict, {"merged_into": primary.memory_id}
-            )
-            memory.updated_at = datetime.now(UTC)
-
-        merged_ids = [m.memory_id for m in duplicates]
-
-        return {
-            "success": True,
-            "primary_memory_id": primary.memory_id,
-            "merged_memory_ids": merged_ids,
-            "created_version_ids": [version.version_id for version in created_versions],
-            "warnings": [],
-            "conflicts": [],
-            "summary": {
-                "merged_count": len(duplicates),
-                "content_merged": strategy.merge_content,
-                "metadata_merged": strategy.merge_metadata,
-            },
-        }
-
-    async def _delete_duplicates(
-        self, memories: list[MemoryRecord], strategy: MergeStrategy
-    ) -> dict[str, Any]:
-        """Permanently delete duplicate memories, keeping the selected primary."""
-        primary = self._select_primary_memory(memories, strategy)
-        duplicates = [m for m in memories if m.memory_id != primary.memory_id]
-        actor = str(strategy.keep_memory_id) if strategy.keep_memory_id else primary.user_id
-
-        created_versions = []
-        for memory in duplicates:
-            snapshot = await self._create_version_snapshot(memory, "Deleted duplicate", actor)
-            created_versions.append(snapshot)
-            await self.db.delete(memory)
-
-        return {
-            "success": True,
-            "primary_memory_id": primary.memory_id,
-            "merged_memory_ids": [m.memory_id for m in duplicates],
-            "created_version_ids": [version.version_id for version in created_versions],
-            "warnings": [],
-            "conflicts": [],
-            "summary": {
-                "deleted_count": len(duplicates),
-                "preserved_memory": primary.memory_id,
-            },
-        }
-
-    async def _archive_duplicates(
-        self, memories: list[MemoryRecord], strategy: MergeStrategy
-    ) -> dict[str, Any]:
-        """Archive duplicate memories without deleting their content."""
-        primary = self._select_primary_memory(memories, strategy)
-        duplicates = [m for m in memories if m.memory_id != primary.memory_id]
-        actor = str(strategy.keep_memory_id) if strategy.keep_memory_id else primary.user_id
-
-        created_versions = []
-        for memory in duplicates:
-            snapshot = await self._create_version_snapshot(memory, "Archived duplicate", actor)
-            created_versions.append(snapshot)
-            memory.status = MemoryStatus.ARCHIVED.value
-            memory.metadata_dict = self._merge_metadata(
-                memory.metadata_dict,
-                {"archived": True, "archived_reason": "deduplication"},
-            )
-            memory.updated_at = datetime.now(UTC)
-
-        return {
-            "success": True,
-            "primary_memory_id": primary.memory_id,
-            "merged_memory_ids": [m.memory_id for m in duplicates],
-            "created_version_ids": [version.version_id for version in created_versions],
-            "warnings": [],
-            "conflicts": [],
-            "summary": {
-                "archived_count": len(duplicates),
-                "preserved_memory": primary.memory_id,
-            },
-        }
-
-    async def _flag_duplicates(
-        self, memories: list[MemoryRecord], _strategy: MergeStrategy
-    ) -> dict[str, Any]:
-        """Flag duplicate memories for manual review without modifying content."""
-        flagged = []
-        for memory in memories:
-            metadata = memory.metadata_dict
-            metadata.setdefault("flags", []).append(
-                {
-                    "reason": "deduplication_review",
-                    "timestamp": datetime.now(UTC).isoformat(),
-                }
-            )
-            memory.metadata_dict = metadata
-            memory.updated_at = datetime.now(UTC)
-            flagged.append(memory)
-
-        return {
-            "success": True,
-            "primary_memory_id": memories[0].memory_id,
-            "merged_memory_ids": [m.memory_id for m in flagged[1:]],
-            "created_version_ids": [],
-            "warnings": ["Manual reviewer attention required"],
-            "conflicts": [],
-            "summary": {
-                "flagged_count": len(flagged),
-                "note": "Duplicates marked for human review",
-            },
-        }
-
-    async def _get_group_memory_ids(self, group_id: uuid.UUID) -> list[uuid.UUID]:
-        """Get memory IDs for a duplicate group."""
-        rows = (
-            (
-                await self.db.execute(
-                    select(MemoryDuplicateMember.memory_id).where(
-                        MemoryDuplicateMember.group_id == group_id
-                    )
-                )
-            )
-            .scalars()
-            .all()
-        )
-        return list(rows)
-
-    async def _persist_duplicate_groups(self, groups: list[DuplicateGroup]) -> None:
-        if not groups:
-            return
-
-        for group in groups:
-            db_group = await self.db.get(MemoryDuplicateGroup, group.group_id)
-            if not db_group:
-                db_group = MemoryDuplicateGroup(id=group.group_id)
-                self.db.add(db_group)
-
-            db_group.representative_memory_id = group.representative_memory_id
-            db_group.similarity_score = group.similarity_score
-            db_group.status = group.status.value
-            metadata = group.metadata or {}
-            metadata["auto_resolvable"] = group.auto_resolvable
-            db_group.group_metadata = metadata
-
-            await self.db.execute(
-                delete(MemoryDuplicateMember).where(
-                    MemoryDuplicateMember.group_id == group.group_id
-                )
-            )
-
-            for memory_id in group.memory_ids:
-                memory_id_str = str(memory_id)
-                member = MemoryDuplicateMember(
-                    group_id=group.group_id,
-                    memory_id=memory_id_str,
-                    similarity_score=group.similarity_score,
-                    is_representative=memory_id == group.representative_memory_id,
-                )
-                self.db.add(member)
-
-        await self.db.commit()
-
-    async def _record_resolution(
-        self,
-        group_id: uuid.UUID,
-        action: DuplicationAction,
-        resolved_by: uuid.UUID,
-        merge_result: dict[str, Any],
-    ) -> None:
-        """Persist resolution metadata for auditing."""
-        group = await self.db.get(MemoryDuplicateGroup, group_id)
-        if not group:
-            return
-
-        metadata = group.group_metadata or {}
-        resolution_metadata = metadata.get("resolution", {})
-        resolution_metadata.update(
-            {
-                "action": action.value if isinstance(action, DuplicationAction) else str(action),
-                "resolved_by": str(resolved_by),
-                "resolved_at": datetime.now(UTC).isoformat(),
-                "summary": merge_result.get("summary"),
-            }
-        )
-        metadata["resolution"] = resolution_metadata
-        group.group_metadata = metadata
-        group.status = DuplicateStatus.RESOLVED.value
-        group.updated_at = datetime.now(UTC)
-
-        # Update membership context for analytics
-        if merge_result.get("merged_memory_ids"):
-            resolved_ids = [str(mem_id) for mem_id in merge_result["merged_memory_ids"]]
-            await self.db.execute(
-                update(MemoryDuplicateMember)
-                .where(
-                    MemoryDuplicateMember.group_id == group_id,
-                    MemoryDuplicateMember.memory_id.in_(resolved_ids),
-                )
-                .values(is_representative=False)
-                .execution_options(synchronize_session=False)
-            )
-
-    async def _calculate_memory_savings(self, groups: list[DuplicateGroup]) -> float:
-        """Estimate reclaimed storage size in megabytes."""
-
-        def _group_member_ids(group: Any) -> list[str]:
-            if hasattr(group, "memory_ids") and group.memory_ids:
-                return [str(mid) for mid in group.memory_ids]
-            if hasattr(group, "members"):
-                return [member.memory_id for member in group.members]
-            return []
-
-        memory_ids = {mem_id for group in groups for mem_id in _group_member_ids(group)}
-        memory_map = await self._load_memories_by_ids(memory_ids)
-
-        reclaimed_bytes = 0
-        for group in groups:
-            representative = str(getattr(group, "representative_memory_id", ""))
-            for mem_id_str in _group_member_ids(group):
-                if representative and mem_id_str == representative:
-                    continue
-                memory = memory_map.get(mem_id_str)
-                if memory and memory.content:
-                    reclaimed_bytes += len(memory.content.encode("utf-8"))
-
-        return reclaimed_bytes / (1024 * 1024)
-
-    async def _load_memories_by_ids(self, memory_ids: set[str]) -> dict[str, MemoryRecord]:
-        if not memory_ids:
-            return {}
-
-        rows = (
-            (
-                await self.db.execute(
-                    select(MemoryRecord).where(MemoryRecord.memory_id.in_(list(memory_ids)))
-                )
-            )
-            .scalars()
-            .all()
-        )
-        return {memory.memory_id: memory for memory in rows}
-
-    # ------------------------------------------------------------------
-    # Phase 229 — Auto-Resolution Engine (FR-229.1)
-    # ------------------------------------------------------------------
-
-    async def auto_resolve(
-        self,
-        session: AsyncSession,
-        *,
-        max_groups: int = 10,
-        cosine_min: float = 1.0,
-        strategy: str = "keep-newest",
-    ) -> AutoResolveResult:
-        """Execute one auto-resolution cycle on detected duplicate groups.
-
-        Detection runs through the existing pipeline (``detect_duplicates``)
-        using ``self.db``; resolution writes (resolutions, tombstones,
-        soft-deletes) target the caller-supplied ``session``.
-
-        NFR-229.2: bounded blast radius — caps at ``max_groups`` per cycle.
-        NFR-229.3: idempotency — skips groups with a resolution in last 24h.
-        NFR-229.4: resolution rows carry kept memory metadata + dropped list.
-        """
-        # 1. Load config (with defaults for missing keys)
-        cfg = self._load_dedup_config()
-
-        # Merge caller overrides with config defaults (guard: 0 is a valid value, not falsy fallback)
-        effective_max = max_groups if max_groups != 10 else cfg.get("max_groups_per_cycle", 10)
-        effective_cosine = cosine_min if cosine_min != 1.0 else cfg.get("cosine_min", 1.0)
-        effective_strategy = strategy if strategy != "keep-newest" else cfg.get("default_strategy", "keep-newest")
-
-        # 2. Run existing detection to find duplicate groups
-        detection_request = DuplicateDetectionRequest(
-            similarity_threshold=effective_cosine,
-            max_results=100,  # Phase 229 perf: reduced from 500 to avoid timeout
-            algorithm=SimilarityAlgorithm.COSINE,
-            include_content=True,
-            include_metadata=False,
-        )
-        detection_result = await self.detect_duplicates(detection_request)
-
-        # 3. Filter and process eligible groups
-        review_groups: list[dict[str, Any]] = []
-        groups_processed = 0
-        memories_resolved = 0
-        memories_tombstoned = 0
-
-        for group in detection_result.duplicate_groups:
-            if groups_processed >= effective_max:
-                review_groups.append(
-                    {
-                        "group_id": str(group.group_id),
-                        "skip_reason": "cycle_cap_reached",
-                        "memory_count": len(group.memory_ids),
-                    }
-                )
-                continue
-
-            # Auto-resolution guard: only groups of exactly 2
-            if len(group.memory_ids) > 2:
-                review_groups.append(
-                    {
-                        "group_id": str(group.group_id),
-                        "skip_reason": "group_size_exceeds_max",
-                        "group_size": len(group.memory_ids),
-                    }
-                )
-                continue
-
-            if len(group.memory_ids) < 2:
-                review_groups.append(
-                    {
-                        "group_id": str(group.group_id),
-                        "skip_reason": "insufficient_members",
-                        "group_size": len(group.memory_ids),
-                    }
-                )
-                continue
-
-            # NFR-229.3: idempotency — skip if resolved in last 24h
-            already_resolved = await self._is_recently_resolved(
-                session, str(group.group_id)
-            )
-            if already_resolved:
-                review_groups.append(
-                    {
-                        "group_id": str(group.group_id),
-                        "skip_reason": "already_resolved_24h",
-                    }
-                )
-                continue
-
-            # 4a. Select keeper based on strategy
-            memory_ids = list(group.memory_ids)
-            mem_map = await self._load_memories_by_ids(set(memory_ids))
-            if len(mem_map) < 2:
-                review_groups.append(
-                    {
-                        "group_id": str(group.group_id),
-                        "skip_reason": "memories_not_found",
-                    }
-                )
-                continue
-
-            keeper_id, dropped_ids_list = self._select_keeper(
-                mem_map, memory_ids, effective_strategy
-            )
-            # keeper_id is used below in the resolution row
-
-            # 4b. Write resolution row
-            resolution = MemoryDedupResolution(
-                group_id=str(group.group_id),
-                kept_id=keeper_id,
-                dropped_ids=dropped_ids_list,
-                strategy=effective_strategy,
-                similarity=group.similarity_score,
-                resolved_by="scheduled",
-                reason=f"Auto-resolved via strategy={effective_strategy}",
-            )
-            session.add(resolution)
-            await session.flush([resolution])
-
-            # 4c. Move dropped memories to tombstones
-            for dropped_id in dropped_ids_list:
-                dropped = mem_map[dropped_id]
-                tombstone = MemoryRecordsTombstone(
-                    original_memory_id=dropped.memory_id,
-                    content=dropped.content or "",
-                    category=dropped.category,
-                    importance=dropped.importance,
-                    external_id=dropped.metadata_dict.get("external_id"),
-                    created_at=dropped.created_at.replace(tzinfo=UTC) if dropped.created_at else None,
-                    resolution_id=resolution.id,
-                    expires_at=(
-                        datetime.now(UTC)
-                        + timedelta(days=cfg.get("tombstone_retention_days", 7))
-                    ),
-                )
-                session.add(tombstone)
-
-            # 4d. Soft-delete dropped memories from memory_records
-            await session.execute(
-                update(MemoryRecord)
-                .where(MemoryRecord.memory_id.in_(dropped_ids_list))
-                .values(
-                    status="deleted",
-                    updated_at=datetime.now(UTC).replace(tzinfo=None),
-                )
-                .execution_options(synchronize_session=False),
-            )
-
-            await session.commit()
-
-            groups_processed += 1
-            memories_resolved += 1  # keeper preserved
-            memories_tombstoned += len(dropped_ids_list)
-
-        return AutoResolveResult(
-            groups_processed=groups_processed,
-            memories_resolved=memories_resolved,
-            memories_tombstoned=memories_tombstoned,
-            review_groups=review_groups,
-            strategy_used=effective_strategy,
-        )
-
-    def _load_dedup_config(self) -> dict[str, Any]:
-        """Load memory_dedup.toml with coded defaults for missing keys."""
-        import tomllib
-
-        config_path = (
-            Path(__file__)
-            .resolve()
-            .parents[3]  # service/  .../vds_memory_orchestrator
-            / "config"
-            / "memory_dedup.toml"
-        )
-        defaults: dict[str, Any] = {
-            "cosine_min": 1.0,
-            "max_groups_per_cycle": 10,
-            "auto_resolve_enabled": False,
-            "tombstone_retention_days": 7,
-            "default_strategy": "keep-newest",
-        }
-        if not config_path.exists():
-            logger.debug("memory_dedup.toml not found at %s; using defaults", config_path)
-            return defaults
-
-        try:
-            with config_path.open("rb") as handle:
-                data = tomllib.load(handle)
-        except Exception as exc:
-            logger.warning("Failed to parse %s: %s", config_path, exc)
-            return defaults
-
-        dedup_section = data.get("dedup", {})
-        if not isinstance(dedup_section, dict):
-            return defaults
-
-        result = dict(defaults)
-        for key in defaults:
-            if key in dedup_section and dedup_section[key] is not None:
-                result[key] = dedup_section[key]
-        return result
-
-    async def _is_recently_resolved(
-        self, session: AsyncSession, group_id: str
-    ) -> bool:
-        """Return True if ``group_id`` has a resolution row within the last 24 hours."""
-        cutoff = datetime.now(UTC) - timedelta(hours=24)
-        stmt = select(func.count(MemoryDedupResolution.id)).where(
-            MemoryDedupResolution.group_id == group_id,
-            MemoryDedupResolution.resolved_at >= cutoff,
-        )
-        count = (await session.execute(stmt)).scalar_one()
-        return count > 0
-
-    @staticmethod
-    def _select_keeper(
-        mem_map: dict[str, MemoryRecord],
-        memory_ids: list[str],
-        strategy: str,
-    ) -> tuple[str, list[str]]:
-        """Select the keeper memory_id and return (keeper_id, [dropped_id, ...])."""
-        if strategy == "keep-most-imported":
-            def _importance_rank(m: MemoryRecord) -> int:
-                return IMPORTANCE_WEIGHTS.get((m.importance or "").lower(), 1)
-
-            keeper_id = max(memory_ids, key=lambda mid: _importance_rank(mem_map[mid]))
-        else:
-            # keep-newest (default) — latest created_at
-            def _created_ts(m: MemoryRecord) -> float:
-                ts = m.created_at or datetime.min.replace(tzinfo=UTC)
-                if hasattr(ts, "timestamp"):
-                    return ts.timestamp()
-                # Handle timezone-aware datetimes
-                if ts.tzinfo is None:
-                    ts = ts.replace(tzinfo=UTC)
-                return ts.timestamp()
-
-            keeper_id = max(memory_ids, key=lambda mid: _created_ts(mem_map[mid]))
-
-        dropped = [mid for mid in memory_ids if mid != keeper_id]
-        return keeper_id, dropped
-
-    # ------------------------------------------------------------------
-    # FR-229.5: tombstone restore
-    # ------------------------------------------------------------------
-
-    async def restore_tombstone(
-        self, session: AsyncSession, memory_id: str
-    ) -> dict[str, Any]:
-        """Restore a tombstoned memory within the 7-day retention window.
-
-        Args:
-            session: DB session for the operation.
-            memory_id: The ``original_memory_id`` stored in the tombstone row.
-
-        Returns:
-            ``{"restored": memory_id, "original_memory_id": ..., "restored_at": ...}``
-
-        Raises:
-            ValueError: If no tombstone is found.
-            ValueError: If the tombstone has expired (``RestoreWindowExpired``).
-        """
-        # 1. Look up tombstone
-        result = await session.execute(
-            select(MemoryRecordsTombstone).where(
-                MemoryRecordsTombstone.original_memory_id == memory_id
-            )
-        )
-        tombstone = result.scalar_one_or_none()
-
-        if tombstone is None:
-            raise ValueError(f"No tombstone found for memory {memory_id}")
-
-        # 2. Check expiry
-        now = datetime.now(UTC)
-        if tombstone.expires_at and tombstone.expires_at.replace(tzinfo=UTC) < now:
-            raise ValueError(
-                f"RestoreWindowExpired: tombstone for {memory_id} "
-                f"expired at {tombstone.expires_at}"
-            )
-
-        # 3. Restore: re-activate the soft-deleted memory record
-        await session.execute(
-            update(MemoryRecord)
-            .where(MemoryRecord.memory_id == memory_id)
-            .values(
-                status="active",
-                updated_at=now.replace(tzinfo=None),
-            )
-            .execution_options(synchronize_session=False),
-        )
-
-        # 4. Remove tombstone
-        await session.delete(tombstone)
-        await session.commit()
-
-        return {
-            "restored": memory_id,
-            "original_memory_id": memory_id,
-            "restored_at": now.isoformat(),
-        }