@ngocsangairvds/vsaf 3.2.15 → 4.0.0

package/tools/vds-scripts/memory_orchestrator/src/vds_memory_orchestrator/service/services/memory_search_service.py

@@ -1,1164 +0,0 @@
-"""Memory search service: Mem0 search, result normalization, and topic clustering."""
-
-import contextlib
-import json
-import logging
-import re
-import time
-from collections import Counter
-from datetime import UTC, datetime, timedelta
-from typing import Any
-from uuid import UUID
-
-import numpy as np
-from sqlalchemy import text
-
-from vds_memory_orchestrator.service.core.ai_memory_exceptions import ServiceUnavailableError
-from vds_memory_orchestrator.service.core.config import REDIS_ANALYTICS_DB
-from vds_memory_orchestrator.service.database.postgres_manager import database_service
-from vds_memory_orchestrator.service.database.redis_manager import redis_manager
-from vds_memory_orchestrator.service.models.ai_memory_models import (
-    Memory,
-    MemorySearchRequest,
-    MemorySearchResponse,
-    TopicCluster,
-    TopicClusteringRequest,
-    TopicClusteringResponse,
-)
-from vds_memory_orchestrator.service.services.storage_strategy import StorageTier
-
-logger = logging.getLogger(__name__)
-
-MS_PER_SECOND = 1000
-
-
-class MemorySearchService:
-    """Encapsulates memory search, result normalization, and topic clustering.
-
-    Receives all collaborators via constructor injection; no module-level singletons
-    are captured on ``self``.  ``database_service`` and ``redis_manager`` are used
-    directly as module-level names so their patch targets live in this module.
-    """
-
-    def __init__(
-        self,
-        config: Any,
-        mem0_sync: Any,
-        redis_service: Any,
-        session_loader: Any = None,
-        semantic_cache: Any = None,
-    ) -> None:
-        self.config = config
-        self._mem0_sync = mem0_sync
-        self._redis_service = redis_service
-        # Optional callable: session_id -> session_data (or None)
-        self._session_loader = session_loader
-        # Phase 223.C: optional semantic cache collaborator for telemetry wiring
-        self._semantic_cache = semantic_cache
-
-    # ------------------------------------------------------------------
-    # Internal helpers
-    # ------------------------------------------------------------------
-
-    @property
-    def _mem0_client(self) -> Any:
-        if self._mem0_sync is not None:
-            return self._mem0_sync._mem0_client
-        return None
-
-    async def _mem0_call(self, method_name: str, **kwargs: Any) -> Any:
-        if self._mem0_sync is not None:
-            return await self._mem0_sync.mem0_call(method_name, **kwargs)
-        raise ServiceUnavailableError("Mem0 client not initialized")
-
-    # ------------------------------------------------------------------
-    # Search
-    # ------------------------------------------------------------------
-
-    async def search_memories(
-        self, request: MemorySearchRequest, include_internal: bool = False
-    ) -> MemorySearchResponse:
-        """Search memories using Mem0 with optional fallback to local search."""
-        try:
-            start_time = datetime.now(UTC)
-
-            # Exclude internal healthcheck/test users from results by default.
-            if (
-                not include_internal
-                and request.user_id
-                and any(request.user_id.startswith(p) for p in ("_healthcheck_", "_test_"))
-            ):
-                return MemorySearchResponse(
-                    query=request.query or "",
-                    results=[],
-                    total_count=0,
-                    limit=request.limit or 10,
-                    offset=request.offset or 0,
-                    cached=False,
-                    search_mode="filtered",
-                )
-
-            def _coerce_list(value: Any) -> list[Any]:
-                if value is None:
-                    return []
-                if isinstance(value, list):
-                    return value
-                return [value]
-
-            def _normalize_results(raw_results: list[Any]) -> list[dict[str, Any]]:
-                normalized: list[dict[str, Any]] = []
-                for item in raw_results or []:
-                    if not isinstance(item, dict):
-                        if hasattr(item, "model_dump") or hasattr(item, "dict"):
-                            item = item.model_dump()
-                        else:
-                            item = {"content": str(item)}
-                    item = dict(item)
-                    item["tags"] = _coerce_list(item.get("tags"))
-                    item["keywords"] = _coerce_list(item.get("keywords"))
-                    item["entities"] = _coerce_list(item.get("entities"))
-                    item["relations"] = _coerce_list(item.get("relations"))
-                    item["consolidation_candidates"] = _coerce_list(
-                        item.get("consolidation_candidates")
-                    )
-                    item["related_memories"] = _coerce_list(item.get("related_memories"))
-                    if item.get("metadata") is None:
-                        item["metadata"] = {}
-                    normalized.append(item)
-                return normalized
-
-            def _parse_datetime(value: Any) -> datetime | None:
-                if value is None:
-                    return None
-                if isinstance(value, datetime):
-                    return value if value.tzinfo else value.replace(tzinfo=UTC)
-                if isinstance(value, str):
-                    try:
-                        if value.endswith("Z"):
-                            value = value.replace("Z", "+00:00")
-                        parsed = datetime.fromisoformat(value)
-                        return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
-                    except Exception:
-                        return None
-                return None
-
-            def _resolve_time_bounds() -> tuple[
-                datetime | None, datetime | None, datetime | None, datetime | None
-            ]:
-                created_after = _parse_datetime(getattr(request, "created_after", None))
-                created_before = _parse_datetime(getattr(request, "created_before", None))
-                updated_after = _parse_datetime(getattr(request, "updated_after", None))
-                updated_before = _parse_datetime(getattr(request, "updated_before", None))
-                preset = getattr(request, "time_range", None)
-                preset_value = getattr(preset, "value", preset)
-                if preset_value:
-                    now = datetime.now(UTC)
-                    if preset_value == "hour":
-                        window = timedelta(hours=1)
-                    elif preset_value == "day":
-                        window = timedelta(days=1)
-                    elif preset_value == "week":
-                        window = timedelta(weeks=1)
-                    elif preset_value == "month":
-                        window = timedelta(days=30)
-                    else:
-                        window = timedelta(days=365)
-                    range_start = now - window
-                    if created_after is None or range_start > created_after:
-                        created_after = range_start
-                    if created_before is None:
-                        created_before = now
-                return created_after, created_before, updated_after, updated_before
-
-            def _passes_time_filters(
-                payload: dict[str, Any],
-                ca: datetime | None,
-                cb: datetime | None,
-                ua: datetime | None,
-                ub: datetime | None,
-            ) -> bool:
-                created_at = _parse_datetime(payload.get("created_at"))
-                updated_at = _parse_datetime(payload.get("updated_at"))
-                if ca and (created_at is not None and created_at < ca):
-                    return False
-                if cb and (created_at is not None and created_at > cb):
-                    return False
-                if ua and (updated_at is not None and updated_at < ua):
-                    return False
-                return not (ub and (updated_at is not None and updated_at > ub))
-
-            # Load session (if provided) for scoping/metadata
-            session_data = None
-            if request.session_id and self._session_loader:
-                try:
-                    session_data = await self._session_loader(request.session_id)
-                except Exception as exc:
-                    logger.warning(
-                        "Failed to load session %s for search context: %s",
-                        request.session_id,
-                        exc,
-                    )
-
-            # Derive org scope (request -> session -> None)
-            org_id = getattr(request, "org_id", None)
-            if org_id is None and session_data and getattr(session_data, "context_metadata", None):
-                org_id = session_data.context_metadata.get("org_id")
-
-            created_after, created_before, updated_after, updated_before = _resolve_time_bounds()
-
-            # Generate cache key
-            cache_prefix = self.config.mem0.read_cache_namespace or "search"
-            project_domain_value = (
-                request.project_domain.value
-                if hasattr(request.project_domain, "value")
-                else request.project_domain
-                if request.project_domain
-                else session_data.project_domain.value
-                if (session_data and getattr(session_data, "project_domain", None))
-                else None
-            )
-
-            cache_key = self._redis_service.generate_cache_key(
-                cache_prefix,
-                org_id=org_id,
-                user_id=request.user_id,
-                session_id=request.session_id,
-                team_id=request.team_id,
-                query=request.query,
-                conversation_id=getattr(request, "conversation_id", None),
-                category=request.category.value
-                if hasattr(request.category, "value")
-                else str(request.category)
-                if request.category
-                else None,
-                importance=request.importance.value
-                if hasattr(request.importance, "value")
-                else str(request.importance)
-                if request.importance
-                else None,
-                tags=request.tags,
-                memory_ids=getattr(request, "memory_ids", None),
-                external_id=getattr(request, "external_id", None),
-                project_domain=project_domain_value,
-                created_after=created_after.isoformat() if created_after else None,
-                created_before=created_before.isoformat() if created_before else None,
-                updated_after=updated_after.isoformat() if updated_after else None,
-                updated_before=updated_before.isoformat() if updated_before else None,
-                time_range=getattr(getattr(request, "time_range", None), "value", None),
-                enable_overfetch=request.enable_overfetch,
-                limit=request.limit,
-                offset=request.offset,
-            )
-
-            # Phase 223.C: semantic cache probe — precedes Redis exact-cache check
-            # so that semantic similarity hits are counted and returned first.
-            if self._semantic_cache is not None and request.query:
-                try:
-                    _sc_hit = await self._semantic_cache.search_with_cache(
-                        query=request.query, user_id=request.user_id or ""
-                    )
-                    if _sc_hit is not None:
-                        _sc_payload = _sc_hit.get("result", {})
-                        if isinstance(_sc_payload, dict):
-                            _sc_payload["cached"] = True
-                            return MemorySearchResponse(**_sc_payload)
-                except Exception as _sc_err:  # pragma: no cover — telemetry best-effort
-                    logger.debug("Semantic cache probe error: %s", _sc_err)
-
-            # Try cache first
-            cached_result = await self._redis_service.get_cached_memory_result(cache_key)
-            if cached_result:
-                logger.info("Memory search cache hit for query: %s", request.query)
-                counters = (
-                    await self._redis_service.get_cache_counters()
-                    if hasattr(self._redis_service, "get_cache_counters")
-                    else {"hits": 1, "misses": 0, "writes": 0}
-                )
-                for res in cached_result.get("results", []):
-                    if not res.get("created_at"):
-                        res["created_at"] = datetime.now(UTC).isoformat()
-                    if not res.get("updated_at"):
-                        res["updated_at"] = res.get("created_at")
-                cached_result["cache_hits"] = counters.get("hits", 0)
-                cached_result["cache_misses"] = counters.get("misses", 0)
-                cached_result["cache_writes"] = counters.get("writes", 0)
-                cached_result["cached"] = True
-                return MemorySearchResponse(**cached_result)
-
-            def _fallback_request() -> MemorySearchRequest:
-                return request.model_copy(
-                    update={
-                        "created_after": created_after,
-                        "created_before": created_before,
-                        "updated_after": updated_after,
-                        "updated_before": updated_before,
-                    }
-                )
-
-            if not self._mem0_client:
-                if self.config.mem0.search_fallback_enabled:
-                    from vds_memory_orchestrator.service.services.fallback_search_service import (
-                        FallbackSearchService,
-                    )
-
-                    fallback_service = FallbackSearchService()
-                    fallback_results = await fallback_service.search(_fallback_request())
-                    normalized_results = _normalize_results(fallback_results.get("results", []))
-                    await self._record_mem0_fallback_metrics(fallback_results.get("search_mode"))
-                    search_results: dict[str, Any] = {
-                        "query": request.query,
-                        "results": normalized_results,
-                        "total_count": fallback_results["total_count"],
-                        "limit": request.limit or 10,
-                        "offset": request.offset or 0,
-                        "search_time_ms": int(
-                            (datetime.now(UTC) - start_time).total_seconds() * MS_PER_SECOND
-                        ),
-                        "cached": False,
-                        "mem0_latency_ms": None,
-                        "search_mode": fallback_results["search_mode"],
-                        "fallback_triggered": True,
-                        "mem0_error": "Mem0 client not initialized",
-                        "cache_hits": 0,
-                        "cache_misses": 0,
-                        "cache_writes": 0,
-                    }
-                    await self._redis_service.cache_memory_result(
-                        cache_key,
-                        search_results,
-                        ttl=self.config.cache.search_results_ttl,
-                    )
-                    return MemorySearchResponse(**search_results)
-                raise ServiceUnavailableError("Mem0 search unavailable")
-
-            effective_user_id = request.user_id or (session_data.user_id if session_data else None)
-
-            search_mode = "mem0"
-            mem0_latency_ms: float | None = None
-            results: list[dict[str, Any]] = []
-            fallback_triggered = False
-            mem0_error: str | None = None
-            fallback_total_count: int | None = None
-
-            if self._mem0_client:
-                try:
-                    # Build Mem0 filters (compact v2 schema)
-                    filters_list: list[dict[str, Any]] = []
-                    if effective_user_id:
-                        filters_list.append({"user_id": effective_user_id})
-                    if request.session_id:
-                        filters_list.append({"metadata": {"session_id": request.session_id}})
-                    if getattr(request, "conversation_id", None):
-                        filters_list.append(
-                            {"metadata": {"conversation_id": request.conversation_id}}
-                        )
-                    if request.project_domain:
-                        filters_list.append(
-                            {
-                                "metadata": {
-                                    "project_domain": getattr(
-                                        request.project_domain,
-                                        "value",
-                                        request.project_domain,
-                                    )
-                                }
-                            }
-                        )
-                    if request.category:
-                        filters_list.append(
-                            {
-                                "metadata": {
-                                    "category": getattr(request.category, "value", request.category)
-                                }
-                            }
-                        )
-                    if request.importance:
-                        filters_list.append(
-                            {
-                                "metadata": {
-                                    "importance": getattr(
-                                        request.importance, "value", request.importance
-                                    )
-                                }
-                            }
-                        )
-                    if request.tags:
-                        filters_list.append({"keywords": {"in": request.tags}})
-                    external_ids: list[str] = []
-                    if getattr(request, "memory_ids", None):
-                        mem0_ids: list[str] = []
-                        for mem_id in request.memory_ids:
-                            mem_id_str = str(mem_id)
-                            if mem_id_str.startswith("pg_"):
-                                external_ids.append(mem_id_str)
-                                continue
-                            try:
-                                mem0_ids.append(str(UUID(mem_id_str)))
-                            except (TypeError, ValueError):
-                                external_ids.append(mem_id_str)
-                        if mem0_ids:
-                            filters_list.append({"memory_ids": mem0_ids})
-                    if getattr(request, "external_id", None):
-                        external_ids.append(request.external_id)
-                    if external_ids:
-                        if len(external_ids) == 1:
-                            filters_list.append({"metadata": {"external_id": external_ids[0]}})
-                        else:
-                            filters_list.append(
-                                {
-                                    "OR": [
-                                        {"metadata": {"external_id": ext_id}}
-                                        for ext_id in external_ids
-                                    ]
-                                }
-                            )
-
-                    if filters_list:
-                        filters: dict[str, Any] = {"AND": filters_list}
-                    else:
-                        user_scope = (
-                            effective_user_id
-                            or request.user_id
-                            or (session_data.user_id if session_data else None)
-                        )
-                        filters = (
-                            {"user_id": user_scope} if user_scope else {"user_id": "agent-001"}
-                        )
-
-                    mem0_start = time.perf_counter()
-                    requested_limit = request.limit or self.config.mem0.search_limit
-                    overfetch = (
-                        min(
-                            requested_limit * self.config.mem0.overfetch_multiplier,
-                            self.config.mem0.max_search_limit,
-                        )
-                        if request.enable_overfetch
-                        else requested_limit
-                    )
-                    search_payload = {
-                        "query": request.query,
-                        "limit": overfetch,
-                        "filters": filters,
-                        "top_k": 100,
-                        "threshold": 0.0,
-                        "rerank": True,
-                    }
-                    mem0_results = await self._mem0_call("search", **search_payload)
-                    mem0_latency_ms = (time.perf_counter() - mem0_start) * MS_PER_SECOND
-
-                    # Mem0Adapter.search() returns list[dict] per VAMI-021/v2.0.0
-                    # contract; legacy {"results": [...]} dict envelope retained
-                    # for cloud-only fallbacks that go through MemoryClient.
-                    raw_results = (
-                        mem0_results
-                        if isinstance(mem0_results, list)
-                        else mem0_results.get("results", [])
-                    )
-                    # If no results, try a relaxed search scoped only by user_id
-                    strict_filters = any(
-                        [
-                            request.session_id,
-                            getattr(request, "conversation_id", None),
-                            request.project_domain,
-                            request.category,
-                            request.importance,
-                            request.tags,
-                            getattr(request, "memory_ids", None),
-                            getattr(request, "external_id", None),
-                        ]
-                    )
-                    if not raw_results and filters and not strict_filters:
-                        relaxed_filters: dict[str, Any] = {}
-                        if effective_user_id:
-                            relaxed_filters["user_id"] = effective_user_id
-                        try:
-                            relaxed_payload = {
-                                "query": request.query,
-                                "limit": overfetch,
-                                "filters": relaxed_filters
-                                if relaxed_filters
-                                else {"user_id": "agent-001"},
-                                "top_k": 100,
-                                "threshold": 0.0,
-                                "rerank": True,
-                            }
-                            mem0_results = await self._mem0_call("search", **relaxed_payload)
-                            raw_results = (
-                                mem0_results
-                                if isinstance(mem0_results, list)
-                                else mem0_results.get("results", [])
-                            )
-                            if raw_results:
-                                search_mode = "mem0-relaxed"
-                        except Exception as relax_exc:  # pragma: no cover
-                            logger.debug("Relaxed Mem0 search failed: %s", relax_exc)
-                    trimmed_results = raw_results[:requested_limit]
-
-                    def _normalize_importance(raw: Any) -> str:
-                        if isinstance(raw, str):
-                            normalized = raw.lower()
-                            if "." in normalized:
-                                normalized = normalized.split(".")[-1]
-                            if normalized in {"low", "medium", "high", "critical"}:
-                                return normalized
-                        return "medium"
-
-                    def _extract_memory_id(result: dict, metadata: dict) -> str:
-                        return (
-                            metadata.get("memory_id")
-                            or metadata.get("external_id")
-                            or result.get("id", "")
-                        )
-
-                    # VAMI-024 B2: fetch canonical metadata from Postgres for
-                    # each returned hit so `category` and `importance` are fresh.
-                    # Mem0's vector-store payloads are write-once at embedding
-                    # time and never refreshed when `update_memory` runs — they
-                    # hold a stale snapshot. Postgres is the canonical source.
-                    fresh_canonical: dict[str, dict[str, Any]] = {}
-                    canonical_ids: set[str] = set()
-                    for result in trimmed_results:
-                        md = result.get("metadata") or {}
-                        mid = _extract_memory_id(result, md)
-                        if mid:
-                            canonical_ids.add(mid)
-                    for mid in canonical_ids:
-                        try:
-                            row = await database_service.get_memory(mid)
-                        except Exception as refresh_exc:  # pragma: no cover
-                            logger.debug(
-                                "Postgres canonical refresh failed for %s: %s",
-                                mid,
-                                refresh_exc,
-                            )
-                            continue
-                        if row is None:
-                            continue
-                        row_meta = getattr(row, "metadata", None) or {}
-                        fresh_canonical[mid] = {
-                            "category": getattr(row, "category", None),
-                            "importance": getattr(row, "importance", None),
-                            "tags": row_meta.get("tags") if isinstance(row_meta, dict) else None,
-                            "session_id": getattr(row, "session_id", None),
-                            "user_id": getattr(row, "user_id", None),
-                            "team_id": getattr(row, "team_id", None),
-                        }
-
-                    # VAMI-024 B1: deduplicate by memory_id. The strict +
-                    # relaxed Mem0 searches (lines ~461/493) can return the
-                    # same underlying memory_id twice because mem0 indexes
-                    # per-fact, not per-memory. Keep the first (highest-score)
-                    # occurrence; drop later duplicates.
-                    seen_memory_ids: set[str] = set()
-                    for result in trimmed_results:
-                        metadata = (result.get("metadata") or {}).copy()
-                        memory_id = _extract_memory_id(result, metadata)
-                        if memory_id and memory_id in seen_memory_ids:
-                            continue
-                        if memory_id:
-                            seen_memory_ids.add(memory_id)
-                        raw_tags = (
-                            metadata.get("tags") or metadata.get("keywords") or result.get("tags")
-                        )
-                        if isinstance(raw_tags, str):
-                            raw_tags = [raw_tags]
-                        tags = raw_tags or []
-                        canonical = fresh_canonical.get(memory_id or "") or {}
-                        session_ref = (
-                            canonical.get("session_id")
-                            or metadata.get("session_id")
-                            or request.session_id
-                            or "mem0_unknown_session"
-                        )
-                        created_at = result.get("created_at") or datetime.now(UTC).isoformat()
-                        updated_at = result.get("updated_at") or created_at
-                        storage_tier = metadata.get("storage_tier") or StorageTier.MEM0_PRO.value
-
-                        normalized_importance = _normalize_importance(
-                            canonical.get("importance") or metadata.get("importance", "medium")
-                        )
-                        metadata["importance"] = normalized_importance
-                        canonical_category = canonical.get("category")
-                        effective_category = (
-                            canonical_category
-                            if canonical_category
-                            else metadata.get("category", "general")
-                        )
-
-                        # VAMI-025 B4: re-apply --category / --importance filters
-                        # against the CANONICAL values. mem0's filter was applied
-                        # at query time against its write-once stale payload; if
-                        # the Postgres row has a different category/importance
-                        # (e.g. the memory was re-categorized after embedding),
-                        # the mem0 hit would leak through despite the fresh
-                        # canonical value not matching the operator's filter.
-                        if request.category:
-                            req_cat = (
-                                request.category.value
-                                if hasattr(request.category, "value")
-                                else str(request.category)
-                            )
-                            if (
-                                effective_category
-                                and str(effective_category).lower() != str(req_cat).lower()
-                            ):
-                                continue
-                        if request.importance:
-                            req_imp = (
-                                request.importance.value
-                                if hasattr(request.importance, "value")
-                                else str(request.importance)
-                            )
-                            if (
-                                normalized_importance
-                                and normalized_importance.lower() != str(req_imp).lower()
-                            ):
-                                continue
-
-                        memory_payload = {
-                            "memory_id": memory_id,
-                            "content": result.get("memory", ""),
-                            "category": effective_category,
-                            "importance": normalized_importance,
-                            "session_id": session_ref,
-                            "user_id": (
-                                canonical.get("user_id")
-                                or metadata.get("user_id")
-                                or result.get("user_id")
-                            ),
-                            "team_id": canonical.get("team_id") or metadata.get("team_id"),
-                            "metadata": {
-                                **metadata,
-                                "mem0_id": result.get("id"),
-                                "mem0_status": metadata.get("mem0_status", "synced"),
-                            },
-                            "created_at": created_at,
-                            "updated_at": updated_at,
-                            "score": result.get("score", 0.0),
-                            "storage_tier": storage_tier,
-                            "tags": canonical.get("tags") or tags,
-                            "keywords": metadata.get("keywords") or [],
-                            "relations": result.get("relations", []),
-                            "embedding": result.get("embedding"),
-                        }
-                        results.append(memory_payload)
-
-                    logger.info(
-                        "Mem0 search completed: %s (%d results, %.2f ms)",
-                        request.query,
-                        len(results),
-                        mem0_latency_ms or -1,
-                    )
-
-                    # Record retrieval telemetry
-                    with contextlib.suppress(Exception):
-                        await self._redis_service.increment_metric(
-                            "mem0_latency_ms_total", int(mem0_latency_ms or 0)
-                        )
-                        await self._redis_service.increment_metric("mem0_latency_samples", 1)
-                        await self._redis_service.increment_metric("mem0_search_requests", 1)
-                        await self._redis_service.increment_metric(
-                            "mem0_overfetch_count", overfetch
-                        )
-                    if self._mem0_sync is not None:
-                        self._mem0_sync._mem0_fail_count = 0
-                except Exception as e:
-                    body = None
-                    try:
-                        resp = getattr(e, "response", None)
-                        if resp is not None:
-                            body = resp.text
-                    except Exception:
-                        body = None
-                    logger.warning("Mem0 search failed (no fallback): %s body=%s", e, body)
-                    mem0_error = str(e)
-                    if self.config.mem0.search_fallback_enabled:
-                        from vds_memory_orchestrator.service.services.fallback_search_service import (
-                            FallbackSearchService,
-                        )
-
-                        fallback_service = FallbackSearchService()
-                        fallback_results = await fallback_service.search(_fallback_request())
-                        await self._record_mem0_fallback_metrics(
-                            fallback_results.get("search_mode")
-                        )
-                        results = fallback_results["results"]
-                        search_mode = fallback_results["search_mode"]
-                        fallback_triggered = True
-                        fallback_total_count = fallback_results["total_count"]
-                    else:
-                        if self._mem0_sync is not None:
-                            self._mem0_sync._mem0_fail_count += 1
-                        raise ServiceUnavailableError("Mem0 search unavailable") from e
-
-            search_time_ms = int((datetime.now(UTC) - start_time).total_seconds() * MS_PER_SECOND)
-
-            def _is_expired(meta: dict[str, Any]) -> bool:
-                exp = meta.get("expires_at")
-                if not exp:
-                    return False
-                try:
-                    return datetime.fromisoformat(exp) < datetime.now(UTC)
-                except Exception:
-                    return False
-
-            normalized_results = _normalize_results(results)
-
-            search_results = {
-                "query": request.query,
-                "results": [
-                    r
-                    for r in normalized_results
-                    if not _is_expired(r.get("metadata") or {})
-                    and _passes_time_filters(
-                        r,
-                        created_after,
-                        created_before,
-                        updated_after,
-                        updated_before,
-                    )
-                ],
-                "total_count": fallback_total_count
-                if fallback_total_count is not None
-                else len(
-                    [
-                        r
-                        for r in normalized_results
-                        if not _is_expired(r.get("metadata") or {})
-                        and _passes_time_filters(
-                            r,
-                            created_after,
-                            created_before,
-                            updated_after,
-                            updated_before,
-                        )
-                    ]
-                ),
-                "limit": request.limit or 10,
-                "offset": request.offset or 0,
-                "search_time_ms": search_time_ms,
-                "cached": False,
-                "mem0_latency_ms": mem0_latency_ms,
-                "search_mode": search_mode or "mem0",
-                "fallback_triggered": fallback_triggered,
-                "mem0_error": mem0_error,
-                "cache_hits": 0,
-                "cache_misses": 0,
-                "cache_writes": 0,
-            }
-
-            # Update search_time_ms with final measurement and cache
-            search_results["search_time_ms"] = int(
-                (datetime.now(UTC) - start_time).total_seconds() * MS_PER_SECOND
-            )
-            search_results["cached"] = False
-
-            counters = {}
-            with contextlib.suppress(Exception):
-                counters = await self._redis_service.get_cache_counters()
-            search_results["cache_hits"] = counters.get("hits", 0)
-            search_results["cache_misses"] = counters.get("misses", 0)
-            search_results["cache_writes"] = counters.get("writes", 0)
-
-            await self._redis_service.cache_memory_result(
-                cache_key,
-                search_results,
-                ttl=self.config.cache.search_results_ttl,
-            )
-
-            # Phase 223.C: populate semantic cache so future probes can hit
-            if self._semantic_cache is not None and request.query:
-                with contextlib.suppress(Exception):
-                    await self._semantic_cache.cache_result(
-                        query=request.query,
-                        result=search_results,
-                        user_id=request.user_id or "",
-                    )
-
-            logger.info(
-                "Memory search completed: %s (%d results)",
-                request.query,
-                len(search_results.get("results", [])),
-            )
-            return MemorySearchResponse(**search_results)
-
-        except ServiceUnavailableError:
-            raise
-        except Exception as e:
-            logger.exception("Error searching memories: %s", e)
-            raise
-
-    # ------------------------------------------------------------------
-    # Clustering
-    # ------------------------------------------------------------------
-
-    async def cluster_memory_topics(
-        self, request: TopicClusteringRequest
-    ) -> TopicClusteringResponse:
-        """Cluster memories into topic groups using embeddings."""
-
-        async def _parse_embedding(raw: Any) -> list[float] | None:
-            if raw is None:
-                return None
-            if isinstance(raw, list):
-                return [float(x) for x in raw]
-            if isinstance(raw, str):
-                stripped = raw.strip().lstrip("[").rstrip("]")
-                if not stripped:
-                    return None
-                try:
-                    return [float(x) for x in stripped.split(",")]
-                except ValueError:
-                    return None
-            return None
-
-        def _snippet(text_: str, limit: int = 160) -> str:
-            clean = " ".join(text_.split())
-            return clean if len(clean) <= limit else f"{clean[:limit].rstrip()}..."
-
-        def _extract_keywords(texts: list[str], tags: list[list[str]]) -> list[str]:
-            stop_words = {
-                "the",
-                "and",
-                "for",
-                "with",
-                "that",
-                "this",
-                "from",
-                "into",
-                "are",
-                "was",
-                "were",
-                "has",
-                "have",
-                "had",
-                "you",
-                "your",
-                "but",
-                "not",
-                "can",
-                "will",
-                "should",
-                "could",
-                "about",
-                "over",
-                "under",
-                "they",
-                "them",
-                "their",
-                "ours",
-                "ourselves",
-                "its",
-                "our",
-            }
-            counts: Counter[str] = Counter()
-            for t in texts:
-                for token in re.findall(r"[a-zA-Z0-9][a-zA-Z0-9_-]{2,}", t.lower()):
-                    if token not in stop_words:
-                        counts[token] += 1
-            for tag_list in tags:
-                for tag in tag_list:
-                    if tag:
-                        counts[tag.lower()] += 2
-            return [word for word, _ in counts.most_common(8)]
-
-        def _kmeans(data: np.ndarray, k: int, max_iter: int) -> np.ndarray:
-            rng = np.random.default_rng(42)
-            indices = rng.choice(data.shape[0], size=k, replace=False)
-            centroids = data[indices]
-            labels = np.zeros(data.shape[0], dtype=int)
-            for _ in range(max_iter):
-                distances = np.linalg.norm(data[:, None, :] - centroids[None, :, :], axis=2)
-                new_labels = distances.argmin(axis=1)
-                if np.array_equal(new_labels, labels):
-                    break
-                labels = new_labels
-                new_centroids = []
-                for idx in range(k):
-                    cluster_points = data[labels == idx]
-                    if cluster_points.size == 0:
-                        new_centroids.append(data[rng.integers(0, data.shape[0])])
-                    else:
-                        new_centroids.append(cluster_points.mean(axis=0))
-                new_centroids = np.stack(new_centroids)
-                if np.linalg.norm(new_centroids - centroids) < 1e-4:
-                    centroids = new_centroids
-                    break
-                centroids = new_centroids
-            return labels
-
-        await database_service.initialize()
-        limit = int(request.limit or 200)
-
-        clauses = ["m.status = 'active'"]
-        params: dict[str, Any] = {"limit": limit}
-        if request.user_id:
-            clauses.append("m.user_id = :user_id")
-            params["user_id"] = request.user_id
-        if request.session_id:
-            clauses.append("m.session_id = :session_id")
-            params["session_id"] = request.session_id
-        if request.team_id:
-            clauses.append("m.team_id = :team_id")
-            params["team_id"] = request.team_id
-        if request.project_domain:
-            params["project_domain"] = getattr(
-                request.project_domain, "value", request.project_domain
-            )
-            clauses.append("m.metadata->>'project_domain' = :project_domain")
-
-        where_sql = " AND ".join(clauses)
-        sql = text(
-            f"""
-            SELECT m.memory_id, m.content, m.metadata, mv.embedding
-            FROM memories m
-            JOIN memory_vectors mv ON mv.memory_id = m.memory_id
-            WHERE {where_sql}
-            ORDER BY m.created_at DESC
-            LIMIT :limit
-            """
-        )
-
-        async with database_service.postgres.engine.begin() as conn:
-            rows = (await conn.execute(sql, params)).mappings().all()
-
-        memory_ids: list[str] = []
-        contents: list[str] = []
-        tags: list[list[str]] = []
-        embeddings: list[list[float]] = []
-
-        for row in rows:
-            embedding = await _parse_embedding(row.get("embedding"))
-            if not embedding:
-                continue
-            memory_ids.append(row.get("memory_id"))
-            contents.append(row.get("content") or "")
-            metadata = row.get("metadata") or {}
-            if isinstance(metadata, str):
-                try:
-                    metadata = json.loads(metadata)
-                except Exception:
-                    metadata = {}
-            raw_tags = metadata.get("tags") if isinstance(metadata, dict) else []
-            if isinstance(raw_tags, str):
-                raw_tags = [raw_tags]
-            tags.append(list(raw_tags) if raw_tags else [])
-            embeddings.append(embedding)
-
-        total_memories = len(memory_ids)
-        if total_memories < 2:
-            return TopicClusteringResponse(
-                total_memories=total_memories,
-                cluster_count=0,
-                unclustered_count=total_memories,
-                clusters=[],
-            )
-
-        data = np.array(embeddings, dtype=float)
-        max_k = min(total_memories, 50)
-        k = request.num_clusters or max(2, int(np.sqrt(total_memories)))
-        k = min(k, max_k)
-
-        labels = _kmeans(data, k, int(request.max_iterations or 50))
-
-        clusters: list[TopicCluster] = []
-        unclustered = 0
-        for cluster_id in range(k):
-            indices = np.where(labels == cluster_id)[0].tolist()
-            if not indices:
-                continue
-            if len(indices) < request.min_cluster_size and not request.include_noise:
-                unclustered += len(indices)
-                continue
-            cluster_memory_ids = [memory_ids[i] for i in indices]
-            cluster_contents = [contents[i] for i in indices]
-            cluster_tags = [tags[i] for i in indices]
-            clusters.append(
-                TopicCluster(
-                    cluster_id=cluster_id,
-                    size=len(indices),
-                    keywords=_extract_keywords(cluster_contents, cluster_tags),
-                    memory_ids=cluster_memory_ids,
-                    sample_memories=[_snippet(t) for t in cluster_contents[:5]],
-                )
-            )
-
-        clusters.sort(key=lambda item: item.size, reverse=True)
-        return TopicClusteringResponse(
-            total_memories=total_memories,
-            cluster_count=len(clusters),
-            unclustered_count=unclustered,
-            clusters=clusters,
-        )
-
-    # ------------------------------------------------------------------
-    # Metrics helper
-    # ------------------------------------------------------------------
-
-    async def _record_mem0_fallback_metrics(self, search_mode: str | None) -> None:
-        """Record Mem0 fallback metrics for monitoring and alerting."""
-        try:
-            async with redis_manager.get_redis(REDIS_ANALYTICS_DB) as conn:
-                await conn.incr("metric:mem0_fallback_requests")
-                if search_mode == "pgvector":
-                    await conn.incr("metric:mem0_fallback_pgvector")
-                elif search_mode == "bm25":
-                    await conn.incr("metric:mem0_fallback_bm25")
-                logger.info(
-                    "Mem0 fallback recorded: mode=%s",
-                    search_mode,
-                )
-        except Exception as exc:
-            logger.warning("Failed to record Mem0 fallback metrics: %s", exc)
-
-    # ------------------------------------------------------------------
-    # Session memory retrieval
-    # ------------------------------------------------------------------
-
-    async def _get_session_memories(self, session_id: str, limit: int = 20) -> list[Memory]:
-        """Get memories for a session with Redis caching + Mem0 fallback."""
-        try:
-            # 1. Try Redis cache first
-            cache_key = f"session:{session_id}:memories:ids"
-            cached_ids = await self._redis_service.get_list(cache_key)
-
-            if cached_ids:
-                logger.info(f"Cache hit for session {session_id}: {len(cached_ids)} memory IDs")
-                memories = []
-                for mem_id in cached_ids[:limit]:
-                    payload_key = f"memory:{mem_id}:payload"
-                    payload = await self._redis_service.get_json(payload_key)
-                    if payload:
-                        payload.setdefault("category", "general")
-                        payload.setdefault("importance", "medium")
-                        try:
-                            memories.append(Memory(**payload))
-                        except Exception as e:
-                            logger.warning(f"Failed to parse memory {mem_id}: {e}")
-                            continue
-
-                if memories:
-                    return memories
-                # Cache had IDs but no payloads — fall through to Mem0
-
-            # 2. Cache miss — fetch from Mem0 if available
-            if not self._mem0_client:
-                logger.warning(f"Mem0 client not available for session {session_id}")
-                return []
-
-            try:
-                logger.info(f"Fetching memories from Mem0 for session {session_id}")
-
-                mem0_results = await self._mem0_call(
-                    "search",
-                    query=f"session:{session_id}",
-                    filters={"AND": [{"run_id": session_id}]},
-                    limit=limit,
-                    top_k=100,
-                    threshold=0.0,
-                    rerank=True,
-                )
-
-                memories = []
-                memory_ids = []
-
-                _results_iter = (
-                    mem0_results
-                    if isinstance(mem0_results, list)
-                    else mem0_results.get("results", [])
-                )
-                for result in _results_iter:
-                    try:
-                        memory = Memory(
-                            memory_id=result.get("id", f"mem_{datetime.now(UTC).timestamp()}"),
-                            content=result.get("memory", result.get("content", "")),
-                            category=result.get("metadata", {}).get("category", "general"),
-                            importance=result.get("metadata", {}).get("importance", "medium"),
-                            session_id=session_id,
-                            user_id=result.get("user_id"),
-                            created_at=result.get("created_at", datetime.now(UTC)),
-                            score=result.get("score", 0.0),
-                        )
-                        memories.append(memory)
-                        memory_ids.append(memory.memory_id)
-
-                        payload_key = f"memory:{memory.memory_id}:payload"
-                        enriched_payload = memory.model_dump()
-                        enriched_payload["relations"] = result.get("relations", [])
-                        enriched_payload["embedding"] = result.get("embedding")
-                        await self._redis_service.set_json(
-                            key=payload_key,
-                            value=enriched_payload,
-                            ttl=self.config.cache.memory_ttl,
-                        )
-                    except Exception as e:
-                        logger.warning(f"Failed to process Mem0 result: {e}")
-                        continue
-
-                if memory_ids:
-                    await self._redis_service.set_list(
-                        cache_key, memory_ids, ttl=self.config.cache.session_ttl
-                    )
-                    logger.info(f"Cached {len(memory_ids)} memory IDs for session {session_id}")
-
-                return memories
-
-            except Exception as e:
-                logger.exception(f"Error fetching from Mem0 for session {session_id}: {e}")
-                return []
-
-        except Exception as e:
-            logger.exception(f"Error in _get_session_memories for {session_id}: {e}")
-            return []
-
-    async def _get_memory_by_id(self, memory_id: str, user_id: str) -> "Memory | None":
-        """Get memory by ID with user permission check."""
-        from vds_memory_orchestrator.service.models.ai_memory_models import Memory
-
-        try:
-            payload_key = f"memory:{memory_id}:payload"
-            payload = await self._redis_service.get_json(payload_key)
-
-            if payload:
-                memory = Memory(**payload)
-                if memory.user_id == user_id:
-                    return memory
-
-            if self._mem0_client:
-                try:
-                    mem0_results = await self._mem0_call(
-                        "search",
-                        query=f"memory_id:{memory_id}",
-                        limit=1,
-                        filters={"user_id": user_id},
-                        top_k=20,
-                        threshold=0.0,
-                        rerank=False,
-                    )
-                    _results_iter = (
-                        mem0_results
-                        if isinstance(mem0_results, list)
-                        else mem0_results.get("results", [])
-                    )
-                    for result in _results_iter:
-                        if result.get("id") == memory_id:
-                            return Memory(
-                                memory_id=result.get("id"),
-                                content=result.get("memory", ""),
-                                relations=result.get("relations", []),
-                                category=result.get("metadata", {}).get("category", "general"),
-                                importance=result.get("metadata", {}).get("importance", "medium"),
-                                session_id=result.get("metadata", {}).get("session_id"),
-                                user_id=result.get("user_id"),
-                                team_id=result.get("metadata", {}).get("team_id"),
-                                created_at=result.get("created_at", datetime.now(UTC)),
-                                score=result.get("score", 0.0),
-                            )
-                except Exception as e:
-                    logger.warning(f"Error fetching from Mem0: {e}")
-
-            return None
-
-        except Exception as e:
-            logger.exception(f"Error getting memory {memory_id}: {e}")
-            return None