@ngocsangairvds/vsaf 3.2.16 → 4.0.0

package/tools/vds-scripts/memory_orchestrator/src/vds_memory_orchestrator/service/services/health_check.py

@@ -1,1556 +0,0 @@
-"""
-VDS AI Memory Service - Enhanced Health Check Service
-Comprehensive health monitoring for triple-layer architecture (Redis + PostgreSQL + Mem0AI).
-
-Phase 1: Redis Infrastructure Implementation
-Health Check Service for Docker Integration
-"""
-
-import logging
-import os
-import time
-from collections.abc import Sequence
-from datetime import UTC, datetime
-from typing import Any, Literal, Union
-
-import httpx
-
-# ──────────────────────────────────────────────────────────────
-# VAMI-013 severity tiering (ADR-3 / ADR-4 / ADR-5)
-# ──────────────────────────────────────────────────────────────
-WarningSeverity = Literal["info", "warning", "critical"]
-WarningEntry = Union[str, dict[str, Any]]  # noqa: UP007
-
-# Explicit substring allowlist (ADR-3): any existing string-warning whose text
-# contains one of these substrings is reclassified as `info` during
-# aggregation. Every entry below has a rationale — do not extend this list
-# without updating the reclassification table in design.md.
-#   1. "Semantic cache hit rate unavailable" — Redis cold-start; emits when
-#      sample_size < SEMANTIC_CACHE_WARNING_MIN_QUERIES (20). Capability
-#      absence, not a fault. Observed in Phase 175 verification.
-#   2. "replay will auto-run every 15m" — memory evolution DLQ pointer; the
-#      underlying `dlq_count > 0` fact is still surfaced as a warning by the
-#      preceding entry. This line is informational guidance, not a signal.
-#   3. "psutil not available" — performance-metrics fallback when optional
-#      dependency is absent. Environment capability, not a fault.
-INFO_WARNING_MATCHERS: tuple[str, ...] = (
-    "Semantic cache hit rate unavailable",
-    "replay will auto-run every 15m",
-    "psutil not available",
-)
-
-
-def _classify_warning(entry: WarningEntry) -> WarningSeverity:
-    """Classify a warning entry as info | warning | critical.
-
-    Rules (per ADR-4 / ADR-5):
-    - ``str`` entries: return ``"info"`` if any substring in
-      :data:`INFO_WARNING_MATCHERS` is present, else ``"warning"``.
-    - ``dict`` entries: honour the ``severity`` key when it is one of
-      ``info | warning | critical``. Otherwise fall back to the safe
-      default ``"warning"`` (malformed input is never silenced).
-    - Any other type (``None``, ``list``, etc.) returns ``"warning"``.
-
-    This helper is pure and side-effect-free; call freely during aggregation.
-    """
-    if isinstance(entry, str):
-        for needle in INFO_WARNING_MATCHERS:
-            if needle in entry:
-                return "info"
-        return "warning"
-    if isinstance(entry, dict):
-        sev = entry.get("severity")
-        if sev in ("info", "warning", "critical"):
-            return sev  # type: ignore[return-value]
-        return "warning"
-    return "warning"
-
-
-def _warning_message(entry: WarningEntry) -> str:
-    """Extract a human-readable message from a warning entry."""
-    if isinstance(entry, str):
-        return entry
-    if isinstance(entry, dict):
-        msg = entry.get("message")
-        if isinstance(msg, str):
-            return msg
-        return str(entry)
-    return str(entry)
-
-
-def _build_warnings_detail(entries: Sequence[WarningEntry]) -> list[dict[str, str]]:
-    """Build a parallel ``warnings_detail`` list of ``{message, severity}``.
-
-    Per ADR-4 this is emitted *alongside* the existing ``warnings[]`` strings;
-    existing consumers that iterate ``warnings[]`` are unaffected.
-
-    ``entries`` is typed as :class:`Sequence` (covariant) rather than
-    :class:`list` (invariant) so callers can pass narrower element types such
-    as ``list[str]`` without triggering Pyright ``reportArgumentType``.
-    """
-    return [
-        {"message": _warning_message(e), "severity": _classify_warning(e)} for e in entries or []
-    ]
-
-
-from sqlalchemy import text
-
-from vds_memory_orchestrator.service.services.memory_evolution import (
-    memory_evolution_tracker,
-)
-from vds_memory_orchestrator.service.services.memory_evolution.celery_app import (
-    CONSOLIDATION_QUEUE_NAME,
-    DEFAULT_QUEUE_NAME,
-)
-
-from ..core.config import (
-    REDIS_ANALYTICS_DB,
-    REDIS_MEMORY_CACHE_DB,
-    get_config,
-)
-from ..database.postgres_manager import get_database_service
-from ..database.redis_manager import get_redis_service, redis_manager
-
-# Note: Connection pooling via Mem0ClientManager was planned but not implemented
-# Using direct Memory client instantiation for health checks
-
-SEMANTIC_CACHE_WARNING_MIN_QUERIES = 20
-EVOLUTION_SLO = {
-    "queue_depth_max": 50,
-    "lag_seconds_max": 300,
-    "dlq_max": 0,
-}
-
-logger = logging.getLogger(__name__)
-
-
-class HealthCheckService:
-    """
-    Comprehensive health check service for VDS AI Memory Service.
-
-    Monitors:
-    - Redis semantic caching layer
-    - PostgreSQL database layer
-    - Mem0AI integration
-    - System performance metrics
-    """
-
-    def __init__(self):
-        self.config = get_config()
-        self._start_time = datetime.now(UTC)
-
-    async def check_all_layers(self) -> dict[str, Any]:
-        """
-        Perform comprehensive health check on all service layers.
-        Called by Docker health check and monitoring systems.
-        """
-        health_report = {
-            "status": "healthy",
-            "timestamp": datetime.now(UTC).isoformat(),
-            "uptime_seconds": (datetime.now(UTC) - self._start_time).total_seconds(),
-            "version": self.config.service.version,
-            "environment": os.getenv("ENVIRONMENT", "development"),
-            "layers": {},
-            "performance": {},
-            "alerts": [],
-        }
-
-        try:
-            # Check Redis Layer
-            redis_health = await self._check_redis_layer()
-            health_report["layers"]["redis"] = redis_health
-
-            # Check PostgreSQL Layer
-            postgres_health = await self._check_postgres_layer()
-            health_report["layers"]["postgres"] = postgres_health
-
-            # Check Mem0AI Layer
-            mem0ai_health = await self._check_mem0ai_layer()
-            health_report["layers"]["mem0ai"] = mem0ai_health
-
-            # Check Performance Metrics
-            performance_metrics = await self._check_performance_metrics()
-            health_report["performance"] = performance_metrics
-
-            # Memory evolution pilot metrics
-            memory_evolution_health = await self._get_memory_evolution_metrics()
-            health_report["layers"]["memory_evolution"] = memory_evolution_health
-
-            # Cache metrics (hit/miss/lock contention)
-            cache_metrics = await self._get_cache_metrics()
-            health_report["layers"]["cache"] = cache_metrics
-
-            # Dedup auto-resolution health (FR-229.4)
-            dedup_health = await self._check_dedup_layer()
-            health_report["layers"]["dedup"] = dedup_health
-
-            # Embedding observability layer (FR-228.4)
-            embedding_health = await self._check_embedding_layer()
-            health_report["layers"]["embedding"] = embedding_health
-
-            # Mem0 sync stall rate rollup (derived from postgres stall metrics)
-            stall_m = postgres_health.get("mem0_stall_metrics", {})
-            stall_rate = stall_m.get("stall_rate_percent", 0.0)
-            if stall_rate < 5.0:
-                sync_status = "healthy"
-            elif stall_rate < 20.0:
-                sync_status = "warning"
-            else:
-                sync_status = "critical"
-
-            # Phase 220.M L-1 / 220.M.1: surface cloud-config + reachability
-            # state from the mem0ai layer so the rollup does not show "healthy"
-            # while the worker is silently skipping every push as
-            # mem0_not_configured. Reuses signals already computed by
-            # _check_mem0ai_layer (mode, circuit_breaker) — no extra probes.
-            #
-            # Escalation contract (Phase 220.M.1, supersedes the unreachable
-            # gamma branch from Phase 220.M):
-            #   - cloud_unreachable=True (configured cloud is runtime-broken)
-            #     escalates sync_status from healthy -> warning.
-            #   - Intentional non-cloud states (self-hosted, unconfigured) do
-            #     NOT escalate — operator did not opt in to cloud sync.
-            cloud_mode = mem0ai_health.get("mode", "unknown")
-            cb = mem0ai_health.get("circuit_breaker") or {}
-            cb_state = cb.get("state", "closed")
-
-            # Resolve cloud_status from mode + circuit_breaker
-            if cloud_mode == "cloud" and cb_state == "open":
-                cloud_status = "unreachable"
-                cloud_unreachable = True  # configured cloud is broken at runtime
-            elif cloud_mode == "cloud":
-                cloud_status = "configured"
-                cloud_unreachable = False
-            elif cloud_mode == "dual":
-                # dual writes to both local + cloud; cloud half can be open
-                cloud_status = "unreachable" if cb_state == "open" else "configured"
-                cloud_unreachable = cb_state == "open"
-            elif cloud_mode == "self-hosted":
-                cloud_status = "not_applicable"  # operator opted for non-cloud
-                cloud_unreachable = False
-            elif cloud_mode == "unconfigured":
-                cloud_status = "not_configured"  # no credentials at all
-                cloud_unreachable = False
-            else:
-                # auto-mode after resolution OR unknown fallback
-                cloud_status = "unknown"
-                cloud_unreachable = False
-
-            # Severity escalation (corrected from gamma-conditional):
-            # Configured-but-runtime-broken (cloud_unreachable=True) escalates
-            # from healthy -> warning. Intentional non-cloud (self-hosted,
-            # unconfigured) does NOT escalate.
-            if sync_status == "healthy" and cloud_unreachable:
-                sync_status = "warning"
-
-            health_report["mem0_sync"] = {
-                "status": sync_status,
-                "cloud_status": cloud_status,
-                "mode": cloud_mode,
-                "stall_rate_percent": stall_rate,
-                "total_memories": stall_m.get("total_memories", 0),
-                "stalled_mem0_count": stall_m.get("stalled_mem0_count", 0),
-                "permanently_stalled_count": stall_m.get("permanently_stalled_count", 0),
-                "stalled_reason_missing_in_list_count": stall_m.get(
-                    "stalled_reason_missing_in_list_count", 0
-                ),
-                "last_reconciliation_at": stall_m.get("last_reconciliation_at"),
-            }
-
-            # Determine overall health status
-            health_report["status"] = self._determine_overall_status(
-                redis_health,
-                postgres_health,
-                mem0ai_health,
-                memory_evolution_health,
-                performance_metrics,
-                dedup_health=dedup_health,
-                embedding_health=embedding_health,
-            )
-
-            # Generate alerts for any issues
-            health_report["alerts"] = self._generate_alerts(health_report)
-            metrics_lines = self._build_metrics_lines(
-                mem0ai_health, memory_evolution_health, cache_metrics
-            )
-            if metrics_lines:
-                # Phase 223.M: use literal backslash-n separator so metrics_text
-                # never carries raw chr(10) newlines.  Raw newlines in a JSON
-                # string field are technically valid after json.dumps escaping,
-                # but they surface as unescaped bytes when responses are
-                # inspected at the HTTP body level (e.g. streaming middleware,
-                # curl | strict-json-parse).  Consumers that previously split on
-                # "\n" should instead split on "\\n".
-                health_report["metrics_text"] = "\\n".join(metrics_lines)
-
-            return health_report
-
-        except Exception as e:
-            logger.exception("Critical error during health check")
-            health_report["status"] = "critical"
-            health_report["error"] = str(e)
-            health_report["alerts"].append(
-                {
-                    "level": "critical",
-                    "message": f"Health check failed: {str(e)}",
-                    "timestamp": datetime.now(UTC).isoformat(),
-                }
-            )
-            return health_report
-
-    async def _check_redis_layer(self) -> dict[str, Any]:
-        """Check Redis layer health and performance."""
-        try:
-            redis_service = await get_redis_service()
-            await redis_service.initialize()
-            redis_health = await redis_service.health_check()
-
-            # Add performance metrics
-            start_time = time.time()
-            async with redis_manager.get_redis(REDIS_MEMORY_CACHE_DB) as conn:
-                await conn.ping()
-            redis_latency = (time.time() - start_time) * 1000
-
-            # Check semantic cache service if available
-            semantic_cache_stats = None
-            try:
-                from .semantic_cache_service import get_semantic_cache_service
-
-                semantic_service = await get_semantic_cache_service()
-                semantic_cache_stats = await semantic_service.get_cache_stats()
-            except Exception as e:
-                logger.debug(f"Semantic cache service not available: {e}")
-
-            redis_layer_health = {
-                "status": redis_health.get("status", "unknown"),
-                "latency_ms": round(redis_latency, 2),
-                "databases": redis_health.get("databases", {}),
-                "semantic_cache": semantic_cache_stats,
-                "performance_targets": {
-                    "max_latency_ms": 10.0,  # Target from DATABASE_ENHANCEMENT_PLAN.md
-                    "min_hit_rate_percent": 70.0,
-                },
-            }
-
-            # Evaluate Redis performance
-            if redis_latency > 50:  # 5x target latency
-                redis_layer_health["status"] = "degraded"
-                redis_layer_health["warnings"] = [f"High latency: {redis_latency:.2f}ms"]
-
-            if semantic_cache_stats:
-                hit_rate = semantic_cache_stats.get("hit_rate_percent", 0)
-                sample_size = semantic_cache_stats.get("total_queries") or 0
-                redis_layer_health["semantic_cache"] = {
-                    "hits": semantic_cache_stats.get("hits", 0),
-                    "misses": semantic_cache_stats.get("misses", 0),
-                    "hit_rate_percent": hit_rate,
-                    "samples": sample_size,
-                }
-                if sample_size >= SEMANTIC_CACHE_WARNING_MIN_QUERIES and hit_rate < 50:
-                    redis_layer_health["status"] = "degraded"
-                    redis_layer_health.setdefault("warnings", []).append(
-                        f"Low cache hit rate: {hit_rate:.1f}% (samples={sample_size})"
-                    )
-                elif sample_size < SEMANTIC_CACHE_WARNING_MIN_QUERIES:
-                    redis_layer_health.setdefault("warnings", []).append(
-                        "Semantic cache hit rate unavailable (insufficient traffic)"
-                    )
-
-            # VAMI-013 / ADR-4: parallel warnings_detail emitted only when the
-            # existing warnings list is present, preserving the legacy shape for
-            # consumers that iterate warnings[].
-            if "warnings" in redis_layer_health:
-                redis_layer_health["warnings_detail"] = _build_warnings_detail(
-                    redis_layer_health["warnings"]
-                )
-
-            return redis_layer_health
-
-        except Exception as e:
-            logger.exception("Error checking Redis layer health")
-            return {
-                "status": "error",
-                "error": str(e),
-                "timestamp": datetime.now(UTC).isoformat(),
-            }
-
-    async def _check_postgres_layer(self) -> dict[str, Any]:
-        """Check PostgreSQL layer health and performance."""
-        try:
-            postgres_service = await get_database_service()
-
-            # Lightweight connectivity check with simple query
-            start_time = time.time()
-            async with postgres_service.postgres.engine.begin() as conn:
-                await conn.execute(text("SELECT 1"))
-            postgres_latency = (time.time() - start_time) * 1000
-
-            postgres_layer_health: dict[str, Any] = {
-                "status": "healthy",
-                "latency_ms": round(postgres_latency, 2),
-                "timestamp": datetime.now(UTC).isoformat(),
-                "connection_pool": {},
-                "performance_targets": {
-                    "max_latency_ms": 50.0,  # Sprint 1 target
-                    "min_connections_available": 5,
-                },
-            }
-
-            # Add stalled/pending counts to health report (monitoring only, not in ping path)
-            stalled_count = 0
-            pending_count = 0
-            stall_metrics: dict[str, Any] = {}
-            try:
-                async with postgres_service.postgres.engine.begin() as conn:
-                    res = await conn.execute(
-                        text(
-                            """
-                            SELECT
-                                COUNT(*) AS total_memories,
-                                COUNT(*) FILTER (WHERE metadata->>'mem0_status' = 'stalled'
-                                    AND (metadata->>'mem0_permanently_stalled')::boolean IS NOT TRUE)
-                                    AS stalled_mem0_count,
-                                COUNT(*) FILTER (WHERE metadata->>'mem0_status' = 'pending')
-                                    AS pending_mem0_count,
-                                COUNT(*) FILTER (
-                                    WHERE metadata->>'mem0_stalled_reason' =
-                                        'mem0_event_succeeded_but_missing_in_list'
-                                    AND (metadata->>'mem0_permanently_stalled')::boolean IS NOT TRUE
-                                ) AS stalled_reason_missing_in_list_count,
-                                COUNT(*) FILTER (
-                                    WHERE (metadata->>'mem0_permanently_stalled')::boolean IS TRUE
-                                ) AS permanently_stalled_count,
-                                MAX(metadata->>'mem0_reconciled_at') AS last_reconciliation_at
-                            FROM memories
-                            """
-                        )
-                    )
-                    row = res.mappings().one()
-                    stalled_count = row["stalled_mem0_count"] or 0
-                    pending_count = row["pending_mem0_count"] or 0
-                    total_memories = row["total_memories"] or 0
-                    stall_rate_pct = (
-                        round(
-                            (stalled_count + (row["stalled_reason_missing_in_list_count"] or 0))
-                            / total_memories
-                            * 100,
-                            2,
-                        )
-                        if total_memories > 0
-                        else 0.0
-                    )
-                    stall_metrics = {
-                        "total_memories": total_memories,
-                        "stalled_mem0_count": stalled_count,
-                        "pending_mem0_count": pending_count,
-                        "stalled_reason_missing_in_list_count": row[
-                            "stalled_reason_missing_in_list_count"
-                        ]
-                        or 0,
-                        "permanently_stalled_count": row["permanently_stalled_count"] or 0,
-                        "last_reconciliation_at": row["last_reconciliation_at"],
-                        "stall_rate_percent": stall_rate_pct,
-                    }
-            except Exception as exc:
-                logger.debug("Stalled count query failed: %s", exc)
-
-            postgres_layer_health["stalled_mem0_count"] = stalled_count
-            postgres_layer_health["pending_mem0_count"] = pending_count
-
-            # Get connection pool stats (monitoring only)
-            pool_stats = {}
-            try:
-                pool = getattr(postgres_service.async_engine, "pool", None)
-                if pool:
-                    capacity = pool.size() if hasattr(pool, "size") else None
-                    checked_out = pool.checked_out() if hasattr(pool, "checked_out") else None
-                    checked_in = pool.checked_in() if hasattr(pool, "checked_in") else None
-                    overflow = pool.overflow() if hasattr(pool, "overflow") else None
-                    available = (
-                        max((capacity or 0) - (checked_out or 0), 0)
-                        if capacity is not None and checked_out is not None
-                        else None
-                    )
-                    pool_stats = {
-                        "size": capacity,
-                        "checked_in": checked_in,
-                        "checked_out": checked_out,
-                        "overflow": overflow,
-                        "available": available,
-                        # FIX 2.1: Show actual pool size settings
-                        "settings": {
-                            "pool_size": getattr(self.config.database, "postgres_pool_size", 10),
-                            "max_overflow": getattr(
-                                self.config.database, "postgres_max_overflow", 20
-                            ),
-                            "pool_timeout": getattr(
-                                self.config.database, "postgres_pool_timeout", 30
-                            ),
-                            "pool_recycle": getattr(
-                                self.config.database, "postgres_pool_recycle", 1800
-                            ),
-                        },
-                    }
-            except Exception:
-                pass
-
-            postgres_layer_health["connection_pool"] = pool_stats
-            postgres_layer_health["pgvector_enabled"] = getattr(
-                postgres_service.postgres, "_pgvector_enabled", False
-            )
-
-            # Evaluate PostgreSQL performance
-            if postgres_latency > postgres_layer_health["performance_targets"]["max_latency_ms"]:
-                postgres_layer_health["status"] = "degraded"
-                postgres_layer_health.setdefault("warnings", []).append(
-                    f"High latency: {postgres_latency:.2f}ms (threshold {postgres_layer_health['performance_targets']['max_latency_ms']}ms)"
-                )
-
-            if stalled_count > 0:
-                postgres_layer_health.setdefault("warnings", []).append(
-                    f"Mem0 stalled promotions: {stalled_count}"
-                )
-            if pending_count > 0:
-                postgres_layer_health.setdefault("warnings", []).append(
-                    f"Mem0 pending promotions: {pending_count}"
-                )
-
-            # FIX 2.1: Updated threshold from 2 to 5 connections (with 30 total, 5 is reasonable)
-            available_connections = pool_stats.get("available")
-            if available_connections is not None and available_connections < 5:
-                postgres_layer_health["status"] = "degraded"
-                postgres_layer_health.setdefault("warnings", []).append(
-                    f"Low available connections: {available_connections} (threshold: 5)"
-                )
-
-            # FIX 4.2: Enhanced connection pool monitoring and alerts
-            pool_size_settings = pool_stats.get("settings", {})
-            max_connections = (
-                pool_size_settings.get("pool_size", 10) + pool_size_settings.get("max_overflow", 20)
-                if pool_stats.get("settings")
-                else 30
-            )
-
-            # Alert on critical pool exhaustion (less than 10% remaining)
-            if available_connections is not None:
-                utilization_percent = (
-                    (max_connections - available_connections) / max_connections
-                ) * 100
-                if utilization_percent > 90:
-                    postgres_layer_health["status"] = "critical"
-                    postgres_layer_health.setdefault("warnings", []).append(
-                        f"CRITICAL: Pool utilization at {utilization_percent:.1f}% "
-                        f"({max_connections - available_connections}/{max_connections} connections in use)"
-                    )
-                elif utilization_percent > 75:
-                    postgres_layer_health.setdefault("warnings", []).append(
-                        f"High pool utilization: {utilization_percent:.1f}% "
-                        f"({max_connections - available_connections}/{max_connections} connections in use)"
-                    )
-
-            # Monitor overflow usage (indicates pool pressure)
-            overflow_count = pool_stats.get("overflow", 0)
-            if overflow_count and overflow_count > 0:
-                max_overflow = pool_size_settings.get("max_overflow", 20)
-                if overflow_count >= max_overflow:
-                    postgres_layer_health["status"] = "critical"
-                    postgres_layer_health.setdefault("warnings", []).append(
-                        f"CRITICAL: Overflow exhausted ({overflow_count}/{max_overflow}), pool under pressure"
-                    )
-                elif overflow_count > (max_overflow * 0.8):
-                    postgres_layer_health.setdefault("warnings", []).append(
-                        f"High overflow usage: {overflow_count}/{max_overflow} connections"
-                    )
-
-            # Add pool health indicators
-            pool_health = {
-                "total_capacity": max_connections,
-                "utilization_percent": round(
-                    ((max_connections - (available_connections or 0)) / max_connections) * 100,
-                    2,
-                )
-                if available_connections is not None
-                else None,
-                "health_status": "healthy",
-            }
-
-            # Determine pool health status
-            if pool_health["utilization_percent"] and pool_health["utilization_percent"] > 90:
-                pool_health["health_status"] = "critical"
-            elif pool_health["utilization_percent"] and pool_health["utilization_percent"] > 75:
-                pool_health["health_status"] = "warning"
-            elif pool_health["utilization_percent"] and pool_health["utilization_percent"] > 50:
-                pool_health["health_status"] = "elevated"
-
-            postgres_layer_health["pool_health"] = pool_health
-            if stall_metrics:
-                postgres_layer_health["mem0_stall_metrics"] = stall_metrics
-
-            # VAMI-013 / ADR-4: parallel warnings_detail (see redis site).
-            if "warnings" in postgres_layer_health:
-                postgres_layer_health["warnings_detail"] = _build_warnings_detail(
-                    postgres_layer_health["warnings"]
-                )
-
-            return postgres_layer_health
-
-        except Exception as e:
-            logger.exception("Error checking PostgreSQL layer health")
-            return {
-                "status": "error",
-                "error": str(e),
-                "timestamp": datetime.now(UTC).isoformat(),
-            }
-
-    async def _check_mem0ai_layer(self) -> dict[str, Any]:
-        """Check Mem0AI integration health with mode-aware backend probes."""
-        mem0_config = self.config.mem0
-
-        # Resolve mode without requiring mem0 package to be installed.
-        try:
-            from vds_memory_orchestrator.service.core.ai_memory_exceptions import (
-                ConfigurationError,
-            )
-            from vds_memory_orchestrator.service.services.mem0_adapter import (
-                _resolve_mode,
-            )
-
-            try:
-                resolved_mode: str = _resolve_mode(mem0_config)
-            except ConfigurationError:
-                resolved_mode = "unconfigured"
-        except Exception:
-            resolved_mode = "cloud" if mem0_config.api_key else "unconfigured"
-
-        mem0_health: dict[str, Any] = {
-            "status": "healthy",
-            "mode": resolved_mode,
-            "configured_features": [],
-            "api_key_configured": bool(mem0_config.api_key),
-            "qdrant_configured": bool(mem0_config.qdrant_host),
-            "byok_enabled": bool(mem0_config.byok_enabled),
-            "read_cache_namespace": mem0_config.read_cache_namespace or "default",
-            "queue_length": None,
-            "performance_targets": {"max_response_time_ms": 1500.0},
-            "latency_ms": None,
-            "http_pool": {
-                "max_connections": mem0_config.http_max_connections,
-                "max_keepalive_connections": mem0_config.http_max_keepalive_connections,
-                "keepalive_expiry_seconds": mem0_config.http_keepalive_expiry,
-                "timeout_seconds": mem0_config.timeout,
-            },
-        }
-
-        if mem0_config.enable_analytics:
-            mem0_health["configured_features"].append("analytics")
-        if mem0_config.byok_enabled:
-            mem0_health["configured_features"].append("byok")
-
-        # Circuit breaker state
-        mem0_health["circuit_breaker"] = {}
-        try:
-            from vds_memory_orchestrator.service.services.ai_memory_service import (
-                ai_memory_service,
-            )
-
-            cb_state = ai_memory_service.get_mem0_circuit_breaker_state()
-            if cb_state:
-                mem0_health["circuit_breaker"] = cb_state
-            else:
-                mem0_health["circuit_breaker"] = {
-                    "state": "closed",
-                    "failures": 0,
-                    "failure_threshold": 5,
-                    "reset_seconds": 60,
-                    "note": "Circuit breaker state not available (service not initialized)",
-                }
-        except Exception as exc:
-            logger.exception("Failed to get circuit breaker state: %s", exc)
-            mem0_health["circuit_breaker"] = {
-                "state": "unknown",
-                "error": str(exc),
-                "failures": 0,
-                "failure_threshold": 5,
-                "reset_seconds": 60,
-            }
-
-        # Queue metrics
-        try:
-            async with redis_manager.get_redis(REDIS_MEMORY_CACHE_DB) as conn:
-                qkeys = await conn.keys("mem0:write_queue:*")
-                mem0_health["queue_length"] = len(qkeys)
-                if len(qkeys) > 0:
-                    mem0_health["status"] = "degraded"
-        except Exception as e:
-            mem0_health["queue_length"] = "unknown"
-            mem0_health["status"] = "degraded"
-            mem0_health["queue_error"] = str(e)
-
-        # Cloud connectivity probe (cloud or dual mode)
-        # Mem0 cloud authenticates with "Authorization: Token <api_key>" (DRF
-        # scheme, NOT Bearer). An unauthenticated GET /v1/ping/ returns 401 even
-        # when the credential is valid, which previously produced a spurious
-        # cloud_reachable:false signal. See phase175 diagnosis.
-        if resolved_mode in ("cloud", "dual") and mem0_config.api_key:
-            try:
-                start_time = time.time()
-                async with httpx.AsyncClient(timeout=5) as hc:
-                    ping_url = f"{mem0_config.host.rstrip('/')}/v1/ping/"
-                    resp = await hc.get(
-                        ping_url,
-                        headers={"Authorization": f"Token {mem0_config.api_key}"},
-                    )
-                    resp.raise_for_status()
-                cloud_latency_ms = (time.time() - start_time) * 1000
-                mem0_health["cloud_latency_ms"] = round(cloud_latency_ms, 2)
-                mem0_health["latency_ms"] = mem0_health["cloud_latency_ms"]
-                if cloud_latency_ms > mem0_health["performance_targets"]["max_response_time_ms"]:
-                    mem0_health.setdefault("warnings", []).append(
-                        f"High cloud API latency: {cloud_latency_ms:.2f}ms "
-                        f"(target {mem0_health['performance_targets']['max_response_time_ms']}ms)"
-                    )
-                    mem0_health["status"] = "degraded"
-            except Exception as e:
-                mem0_health["cloud_reachable"] = False
-                mem0_health.setdefault("warnings", []).append(f"Cloud ping failed: {e}")
-                if resolved_mode == "cloud":
-                    mem0_health["status"] = "degraded"
-
-        # Qdrant + Ollama probes (self-hosted or dual mode)
-        if resolved_mode in ("self-hosted", "dual") and mem0_config.qdrant_host:
-            qdrant_base = f"http://{mem0_config.qdrant_host}:{mem0_config.qdrant_port}"
-            try:
-                async with httpx.AsyncClient(timeout=3) as hc:
-                    resp = await hc.get(f"{qdrant_base}/")
-                mem0_health["qdrant_reachable"] = resp.status_code < 500
-            except Exception as e:
-                mem0_health["qdrant_reachable"] = False
-                mem0_health.setdefault("warnings", []).append(f"Qdrant unreachable: {e}")
-                if resolved_mode == "self-hosted":
-                    mem0_health["status"] = "degraded"
-
-            if (
-                os.environ.get("MEM0_OLLAMA_ENABLED", "").lower() == "true"
-                or os.environ.get("MEM0_LLM_PROVIDER", "") == "ollama"
-            ):
-                ollama_base = mem0_config.embedder_base_url.rstrip("/")
-                try:
-                    async with httpx.AsyncClient(timeout=3) as hc:
-                        resp = await hc.get(f"{ollama_base}/api/tags")
-                    mem0_health["ollama_reachable"] = resp.status_code < 500
-                except Exception as e:
-                    mem0_health["ollama_reachable"] = False
-                    mem0_health.setdefault("warnings", []).append(f"Ollama unreachable: {e}")
-            else:
-                mem0_health["ollama_reachable"] = (
-                    None  # ADR-60: Ollama probe skipped; gate on MEM0_OLLAMA_ENABLED=true or MEM0_LLM_PROVIDER=ollama
-                )
-
-        # Circuit breaker open check
-        cb_state_val = (
-            mem0_health.get("circuit_breaker", {}).get("state")
-            if isinstance(mem0_health.get("circuit_breaker"), dict)
-            else None
-        )
-        if cb_state_val == "open":
-            mem0_health.setdefault("warnings", []).append("Mem0 circuit breaker open")
-            if mem0_health.get("status") != "error":
-                mem0_health["status"] = "degraded"
-
-        # VAMI-013 / ADR-4: parallel warnings_detail (see redis site).
-        if "warnings" in mem0_health:
-            mem0_health["warnings_detail"] = _build_warnings_detail(mem0_health["warnings"])
-
-        return mem0_health
-
-    async def _check_dedup_layer(self) -> dict[str, Any]:
-        """Check dedup auto-resolution health (FR-229.4)."""
-        dedup_health: dict[str, Any] = {
-            "status": "healthy",
-            "dedup_groups": 0,
-            "dedup_dups_total": 0,
-            "dedup_resolved_24h": 0,
-            "dedup_resolution_strategy_breakdown": {},
-        }
-
-        try:
-            postgres_service = await get_database_service()
-
-            # Query duplicate groups count and total members
-            async with postgres_service.postgres.engine.begin() as conn:
-                res = await conn.execute(
-                    text(
-                        """
-                        SELECT
-                            COUNT(*) AS dedup_groups,
-                            COALESCE(SUM(
-                                (SELECT COUNT(*) FROM memory_duplicate_members mdm
-                                 WHERE mdm.group_id = mdg.id)
-                            ), 0) AS dedup_dups_total
-                        FROM memory_duplicate_groups mdg
-                        """
-                    )
-                )
-                row = res.mappings().one()
-                dedup_health["dedup_groups"] = row["dedup_groups"] or 0
-                dedup_health["dedup_dups_total"] = row["dedup_dups_total"] or 0
-
-            # Query resolved in last 24h and strategy breakdown
-            async with postgres_service.postgres.engine.begin() as conn:
-                res = await conn.execute(
-                    text(
-                        """
-                        SELECT
-                            COUNT(*) AS dedup_resolved_24h
-                        FROM memory_dedup_resolutions
-                        WHERE resolved_at > NOW() - INTERVAL '24 hours'
-                        """
-                    )
-                )
-                row = res.mappings().one()
-                dedup_health["dedup_resolved_24h"] = row["dedup_resolved_24h"] or 0
-
-            # Strategy breakdown from last 7 days
-            async with postgres_service.postgres.engine.begin() as conn:
-                res = await conn.execute(
-                    text(
-                        """
-                        SELECT
-                            strategy,
-                            COUNT(*) AS count
-                        FROM memory_dedup_resolutions
-                        WHERE resolved_at > NOW() - INTERVAL '7 days'
-                        GROUP BY strategy
-                        """
-                    )
-                )
-                strategy_breakdown: dict[str, int] = {}
-                for strategy_row in res.mappings():
-                    strategy_breakdown[str(strategy_row["strategy"])] = int(
-                        strategy_row["count"]
-                    )
-                dedup_health["dedup_resolution_strategy_breakdown"] = strategy_breakdown
-
-            # FR-229.4: escalate when dups exist but no auto-resolution is happening
-            if (
-                dedup_health["dedup_dups_total"] > 100
-                and dedup_health["dedup_resolved_24h"] == 0
-            ):
-                dedup_health["status"] = "degraded"
-                dedup_health["reason"] = "dedup auto-resolution stalled"
-                dedup_health.setdefault("warnings", []).append(
-                    f"dedup auto-resolution stalled: {dedup_health['dedup_dups_total']} duplicates "
-                    f"across {dedup_health['dedup_groups']} groups with 0 resolved in last 24h"
-                )
-
-            # VAMI-013 / ADR-4: parallel warnings_detail
-            if "warnings" in dedup_health:
-                dedup_health["warnings_detail"] = _build_warnings_detail(
-                    dedup_health["warnings"]
-                )
-
-            return dedup_health
-
-        except Exception as e:
-            logger.exception("Error checking dedup layer health")
-            return {
-                "status": "error",
-                "error": str(e),
-                "timestamp": datetime.now(UTC).isoformat(),
-            }
-
-    async def _check_embedding_layer(self) -> dict[str, Any]:
-        """Check embedding provider health (FR-228.4, FR-228.5)."""
-        import os
-
-        embedding_health: dict[str, Any] = {
-            "status": "healthy",
-            "provider": os.environ.get("VDS_AGENT_CORE_EMBED__PROVIDER", "ollama"),
-            "model": os.environ.get("VDS_AGENT_CORE_EMBED__MODEL", "bge-m3"),
-            "dimension": os.environ.get("VDS_AGENT_CORE_EMBED__DIMENSION", "1024"),
-            "base_url": os.environ.get(
-                "VDS_AGENT_CORE_EMBED__BASE_URL", "http://host.docker.internal:11434"
-            ),
-            "verify_dim": os.environ.get(
-                "VDS_AGENT_CORE_EMBED__VERIFY_DIM_AGAINST_QDRANT", "true"
-            ),
-            "endpoint_reachable": None,
-            "latency_ms": None,
-        }
-
-        verify_dim = embedding_health["verify_dim"].lower() in ("true", "1", "yes", "")
-        provider = embedding_health["provider"]
-
-        # FR-228.5: degraded when verify_dim is disabled
-        if not verify_dim:
-            embedding_health["status"] = "degraded"
-            embedding_health["reason"] = "embedding dim verification disabled"
-            embedding_health.setdefault("warnings", []).append(
-                "Embedding dimension verification disabled (NFR-225.11)"
-            )
-
-        # FR-228.4: Ollama reachability probe (gated by HEALTH_PROBE_ENABLED)
-        health_probe = os.environ.get(
-            "VDS_AGENT_CORE_EMBED__HEALTH_PROBE_ENABLED", "true"
-        )
-        if provider == "ollama" and health_probe.lower() in ("true", "1", "yes", ""):
-            import time as _time
-
-            timeout_s = float(
-                os.environ.get("VDS_AGENT_CORE_EMBED__HEALTH_PROBE_TIMEOUT_MS", "2000")
-            ) / 1000.0
-            base_url = embedding_health["base_url"].rstrip("/")
-            try:
-                start = _time.time()
-                async with httpx.AsyncClient(timeout=timeout_s) as client:
-                    resp = await client.get(f"{base_url}/api/tags")
-                embedding_health["endpoint_reachable"] = resp.status_code < 500
-                embedding_health["latency_ms"] = round(
-                    (_time.time() - start) * 1000, 2
-                )
-                if not embedding_health["endpoint_reachable"]:
-                    embedding_health["status"] = "degraded"
-                    embedding_health.setdefault("warnings", []).append(
-                        f"Ollama returned HTTP {resp.status_code}"
-                    )
-            except Exception as e:
-                embedding_health["endpoint_reachable"] = False
-                # FR-228.5: degraded when endpoint unreachable
-                if embedding_health.get("status") == "healthy":
-                    embedding_health["status"] = "degraded"
-                embedding_health.setdefault("warnings", []).append(
-                    f"Ollama unreachable: {e}"
-                )
-
-        if "warnings" in embedding_health:
-            embedding_health["warnings_detail"] = _build_warnings_detail(
-                embedding_health["warnings"]
-            )
-
-        return embedding_health
-
-    async def _get_memory_evolution_metrics(self) -> dict[str, Any]:
-        """Summarize memory evolution activity for pilot monitoring."""
-
-        tracker_settings = getattr(memory_evolution_tracker, "_settings", None)
-        shadow_mode = tracker_settings.shadow_mode if tracker_settings else True
-        enabled = tracker_settings.enabled if tracker_settings else False
-        pilot_orgs = sorted(set(self.config.service.memory_evolution_pilot_orgs or []))
-        default_org = self.config.service.default_org_id or "default"
-        recent_window_minutes = 60
-
-        try:
-            db_service = await get_database_service()
-            async with db_service.get_session() as session:
-                query = text(
-                    """
-                    SELECT
-                        COALESCE(NULLIF(org_id, ''), :default_org) AS org_id,
-                        COUNT(*) AS total_events,
-                        COUNT(*) FILTER (
-                            WHERE created_at >= NOW() - (:window_minutes * INTERVAL '1 minute')
-                        ) AS recent_events,
-                        COUNT(*) FILTER (WHERE processing_state = 'blocked') AS blocked_events,
-                        MIN(created_at) AS first_seen_at,
-                        MAX(created_at) AS last_seen_at
-                    FROM memory_evolution_events
-                    GROUP BY 1
-                    ORDER BY 1
-                    """
-                )
-                result = await session.execute(
-                    query,
-                    {
-                        "default_org": default_org,
-                        "window_minutes": recent_window_minutes,
-                    },
-                )
-
-                action_query = text(
-                    """
-                    SELECT
-                        COUNT(*) AS total_actions,
-                        COUNT(*) FILTER (
-                            WHERE created_at >= NOW() - (:window_minutes * INTERVAL '1 minute')
-                        ) AS recent_actions,
-                        COUNT(*) FILTER (
-                            WHERE jsonb_array_length(
-                                COALESCE(metadata -> 'conflict_memory_ids', '[]'::jsonb)
-                            ) > 0
-                        ) AS conflict_actions,
-                        MAX(created_at) AS last_action_at
-                    FROM memory_evolution_actions
-                    """
-                )
-                action_result = await session.execute(
-                    action_query,
-                    {"window_minutes": recent_window_minutes},
-                )
-                action_row = action_result.mappings().first()
-
-                org_stats = []
-                recent_non_pilot = 0
-                for row in result.mappings():
-                    org_id = row["org_id"] or default_org
-                    is_pilot = tracker_settings.is_pilot_org(org_id) if tracker_settings else False
-                    stat = {
-                        "org_id": org_id,
-                        "total_events": int(row["total_events"] or 0),
-                        "recent_events": int(row["recent_events"] or 0),
-                        "blocked_events": int(row["blocked_events"] or 0),
-                        "first_seen_at": row["first_seen_at"].isoformat()
-                        if row["first_seen_at"]
-                        else None,
-                        "last_seen_at": row["last_seen_at"].isoformat()
-                        if row["last_seen_at"]
-                        else None,
-                        "pilot": is_pilot,
-                    }
-                    org_stats.append(stat)
-                    if not is_pilot and stat["recent_events"]:
-                        recent_non_pilot += stat["recent_events"]
-
-                blocked_stmt = text(
-                    """
-                    SELECT COUNT(*)
-                    FROM memory_evolution_events
-                    WHERE processing_state = 'blocked'
-                      AND created_at >= NOW() - (:window_minutes * INTERVAL '1 minute')
-                    """
-                )
-                blocked_result = await session.execute(
-                    blocked_stmt,
-                    {"window_minutes": recent_window_minutes},
-                )
-                blocked_events = int(blocked_result.scalar() or 0)
-
-                # Pending lag and DLQ
-                lag_stmt = text(
-                    """
-                    SELECT EXTRACT(EPOCH FROM (NOW() - MIN(created_at))) AS lag_seconds
-                    FROM memory_evolution_events
-                    WHERE processing_state IN ('pending','shadow')
-                    """
-                )
-                lag_result = await session.execute(lag_stmt)
-                pending_lag_seconds = float(lag_result.scalar() or 0)
-
-                # Prefer live Redis DLQ length; fall back to DB table count
-                dlq_count = 0
-                try:
-                    redis = await get_redis_service()
-                    dlq_count = await redis.llen("memory_evolution_failures")
-                except Exception:  # pragma: no cover - best effort
-                    logger.debug("Redis DLQ length unavailable; falling back to DB")
-                    dlq_count = 0
-
-                dlq_stmt = text("SELECT COUNT(*) FROM memory_evolution_failures")
-                dlq_result = await session.execute(dlq_stmt)
-                db_dlq = int(dlq_result.scalar() or 0)
-                dlq_count = max(dlq_count, db_dlq)
-
-                sweep_stmt = text(
-                    """
-                    SELECT conflict_actions, queue_depth_max, conflict_anomaly, queue_anomaly, completed_at
-                    FROM memory_evolution_sweep_reports
-                    ORDER BY completed_at DESC
-                    LIMIT 1
-                    """
-                )
-                sweep_result = await session.execute(sweep_stmt)
-                sweep_row = sweep_result.mappings().first()
-                latest_sweep = None
-                if sweep_row:
-                    latest_sweep = {
-                        "conflict_actions": int(sweep_row["conflict_actions"] or 0),
-                        "queue_depth_max": int(sweep_row["queue_depth_max"] or 0),
-                        "conflict_anomaly": bool(sweep_row["conflict_anomaly"]),
-                        "queue_anomaly": bool(sweep_row["queue_anomaly"]),
-                        "completed_at": sweep_row["completed_at"].isoformat()
-                        if sweep_row["completed_at"]
-                        else None,
-                    }
-
-            queue_stats = await self._get_celery_queue_stats()
-
-            status = "shadow" if shadow_mode else "pilot"
-            warnings: list[str] = []
-            if recent_non_pilot > 0 and pilot_orgs:
-                status = "degraded"
-                warnings.append(f"{recent_non_pilot} recent events recorded for non-pilot orgs")
-
-            for queue_name, data in queue_stats.items():
-                depth = data.get("depth")
-                if isinstance(depth, int) and depth > EVOLUTION_SLO["queue_depth_max"]:
-                    warnings.append(
-                        f"Queue {queue_name} depth {depth} exceeds SLO {EVOLUTION_SLO['queue_depth_max']}"
-                    )
-
-            if pending_lag_seconds > EVOLUTION_SLO["lag_seconds_max"]:
-                warnings.append(
-                    f"Pending lag {pending_lag_seconds:.0f}s exceeds SLO {EVOLUTION_SLO['lag_seconds_max']}s"
-                )
-            if dlq_count > EVOLUTION_SLO["dlq_max"]:
-                warnings.append(f"DLQ count {dlq_count} exceeds SLO {EVOLUTION_SLO['dlq_max']}")
-                # Raise health status to degraded and surface actionable hint
-                status = "degraded"
-                warnings.append(
-                    "Memory evolution DLQ is non-empty; replay will auto-run every 15m via beat or trigger /tasks/replay-dlq"
-                )
-
-            wildcard_active = "*" in (tracker_settings.pilot_orgs if tracker_settings else frozenset())
-            if wildcard_active:
-                status = "all"
-
-            return {
-                "status": status if enabled else "disabled",
-                "wildcard_active": wildcard_active,
-                "shadow_mode": shadow_mode,
-                "enabled": enabled,
-                "pilot_orgs": pilot_orgs,
-                "default_org_id": default_org,
-                "recent_window_minutes": recent_window_minutes,
-                "org_statistics": org_stats,
-                "non_pilot_recent_events": recent_non_pilot,
-                "warnings": warnings,
-                # VAMI-013 / ADR-4: parallel warnings_detail (see redis site).
-                "warnings_detail": _build_warnings_detail(warnings),
-                "actions": {
-                    "total": int(action_row["total_actions"] or 0) if action_row else 0,
-                    "recent": int(action_row["recent_actions"] or 0) if action_row else 0,
-                    "conflicts": int(action_row["conflict_actions"] or 0) if action_row else 0,
-                    "last_action_at": (
-                        action_row["last_action_at"].isoformat()
-                        if action_row and action_row["last_action_at"]
-                        else None
-                    ),
-                },
-                "queues": queue_stats,
-                "blocked_events": blocked_events,
-                "pending_lag_seconds": pending_lag_seconds,
-                "dlq_count": dlq_count,
-                "latest_sweep": latest_sweep,
-            }
-        except Exception as exc:
-            logger.debug("Memory evolution metrics unavailable: %s", exc)
-            return {
-                "status": "unknown",
-                "error": str(exc),
-                "shadow_mode": shadow_mode,
-                "enabled": enabled,
-                "pilot_orgs": pilot_orgs,
-                "default_org_id": default_org,
-                "recent_window_minutes": recent_window_minutes,
-                "org_statistics": [],
-            }
-
-    async def _get_celery_queue_stats(self) -> dict[str, dict[str, Any]]:
-        stats: dict[str, dict[str, Any]] = {}
-        try:
-            async with redis_manager.get_redis(REDIS_MEMORY_CACHE_DB) as conn:
-                for queue_name in (DEFAULT_QUEUE_NAME, CONSOLIDATION_QUEUE_NAME):
-                    depth = await conn.llen(queue_name)
-                    stats[queue_name] = {"depth": int(depth or 0)}
-        except Exception as exc:
-            logger.debug("Unable to read Celery queue depth: %s", exc)
-            stats["error"] = {"message": str(exc)}
-        return stats
-
-    @staticmethod
-    def _build_metrics_lines(
-        mem0_health: dict[str, Any],
-        memory_evolution_health: dict[str, Any],
-        cache_metrics: dict[str, Any],
-    ) -> list[str]:
-        lines: list[str] = []
-        if isinstance(mem0_health, dict):
-            queue_length = mem0_health.get("queue_length")
-            if isinstance(queue_length, int):
-                lines.append(f"mem0_write_queue_length {queue_length}")
-            latency = mem0_health.get("api_latency_ms")
-            if isinstance(latency, (int, float)):
-                lines.append(f"mem0_api_latency_ms {latency}")
-            status_degraded = 1 if mem0_health.get("status") == "degraded" else 0
-            lines.append(f"mem0_status_degraded {status_degraded}")
-            if mem0_health.get("byok_enabled") is not None:
-                lines.append(f"mem0_byok_enabled {1 if mem0_health.get('byok_enabled') else 0}")
-            cb_state = mem0_health.get("circuit_breaker")
-            if isinstance(cb_state, dict):
-                state = cb_state.get("state")
-                if state:
-                    lines.append(f"mem0_circuit_open {1 if state == 'open' else 0}")
-                failures = cb_state.get("failures")
-                if isinstance(failures, int):
-                    lines.append(f"mem0_circuit_failures {failures}")
-                remaining = cb_state.get("remaining_seconds")
-                if isinstance(remaining, (int, float)):
-                    lines.append(f"mem0_circuit_remaining_seconds {remaining}")
-
-        if isinstance(memory_evolution_health, dict):
-            actions = memory_evolution_health.get("actions") or {}
-            conflicts = actions.get("conflicts")
-            if isinstance(conflicts, int):
-                lines.append(f"memory_evolution_conflict_actions {conflicts}")
-            queues = memory_evolution_health.get("queues") or {}
-            for queue_name, queue_data in queues.items():
-                depth = queue_data.get("depth")
-                if isinstance(depth, int):
-                    lines.append(f'memory_evolution_queue_depth{{queue="{queue_name}"}} {depth}')
-                    if depth > EVOLUTION_SLO["queue_depth_max"]:
-                        lines.append(
-                            f'memory_evolution_queue_slo_violations{{queue="{queue_name}"}} 1'
-                        )
-            blocked = memory_evolution_health.get("blocked_events", 0)
-            if isinstance(blocked, int):
-                lines.append(f"memory_evolution_blocked_events {blocked}")
-            lag = memory_evolution_health.get("pending_lag_seconds")
-            if isinstance(lag, (int, float)):
-                lines.append(f"memory_evolution_pending_lag_seconds {lag}")
-                if lag > EVOLUTION_SLO["lag_seconds_max"]:
-                    lines.append("memory_evolution_lag_slo_violations 1")
-            dlq = memory_evolution_health.get("dlq_count")
-            if isinstance(dlq, int):
-                lines.append(f"memory_evolution_dlq_count {dlq}")
-                if dlq > EVOLUTION_SLO["dlq_max"]:
-                    lines.append("memory_evolution_dlq_slo_violations 1")
-            latest_sweep = memory_evolution_health.get("latest_sweep") or {}
-            sweep_depth = latest_sweep.get("queue_depth_max")
-            if isinstance(sweep_depth, int):
-                lines.append(f"memory_evolution_latest_sweep_queue_depth_max {sweep_depth}")
-            sweep_conflicts = latest_sweep.get("conflict_actions")
-            if isinstance(sweep_conflicts, int):
-                lines.append(f"memory_evolution_latest_sweep_conflict_actions {sweep_conflicts}")
-            for flag_name in ("conflict_anomaly", "queue_anomaly"):
-                if flag_name in latest_sweep:
-                    lines.append(
-                        f"memory_evolution_latest_sweep_{flag_name} "
-                        f"{1 if latest_sweep.get(flag_name) else 0}"
-                    )
-
-        # ADR-57 Option-B: cache_search_hits/writes removed; re-introduce when mem0 cloud p99 >5s OR query-rate >100/min
-        if isinstance(cache_metrics, dict):
-            if "search_cache_misses" in cache_metrics:
-                lines.append(f"cache_search_misses {cache_metrics['search_cache_misses']}")
-            if "analytics_lock_contention" in cache_metrics:
-                lines.append(
-                    f"cache_analytics_lock_contention {cache_metrics['analytics_lock_contention']}"
-                )
-
-            # Retrieval telemetry (Mem0)
-            for key, metric_name in (
-                ("mem0_latency_samples", "mem0_latency_samples"),
-                ("mem0_latency_total_ms", "mem0_latency_total_ms"),
-                ("mem0_search_requests", "mem0_search_requests"),
-                ("mem0_rerank_requests", "mem0_rerank_requests"),
-                ("mem0_overfetch_count", "mem0_overfetch_count"),
-            ):
-                if key in cache_metrics:
-                    lines.append(f"{metric_name} {cache_metrics[key]}")
-            for key in (
-                "mem0_fallback_requests",
-                "mem0_fallback_pgvector",
-                "mem0_fallback_bm25",
-            ):
-                if key in cache_metrics:
-                    lines.append(f"{key} {cache_metrics[key]}")
-
-        return lines
-
-    async def _check_performance_metrics(self) -> dict[str, Any]:
-        """Check system performance metrics."""
-        try:
-            import psutil
-
-            # System metrics
-            cpu_percent = psutil.cpu_percent(interval=1)
-            memory = psutil.virtual_memory()
-            disk = psutil.disk_usage("/")
-
-            performance_metrics = {
-                "system": {
-                    "cpu_percent": cpu_percent,
-                    "memory_percent": memory.percent,
-                    "memory_available_gb": round(memory.available / (1024**3), 2),
-                    "disk_percent": disk.percent,
-                    "disk_free_gb": round(disk.free / (1024**3), 2),
-                },
-                "service": {
-                    "uptime_seconds": (datetime.now(UTC) - self._start_time).total_seconds(),
-                    "active_connections": 0,  # Would be populated from actual metrics
-                    "requests_per_minute": 0,  # Would be populated from actual metrics
-                },
-                "targets": {
-                    "max_cpu_percent": 80.0,
-                    "max_memory_percent": 85.0,
-                    "max_disk_percent": 90.0,
-                },
-            }
-
-            # Evaluate performance against targets
-            performance_issues = []
-            if cpu_percent > performance_metrics["targets"]["max_cpu_percent"]:
-                performance_issues.append(f"High CPU usage: {cpu_percent}%")
-
-            if memory.percent > performance_metrics["targets"]["max_memory_percent"]:
-                performance_issues.append(f"High memory usage: {memory.percent}%")
-
-            if disk.percent > performance_metrics["targets"]["max_disk_percent"]:
-                performance_issues.append(f"High disk usage: {disk.percent}%")
-
-            if performance_issues:
-                performance_metrics["warnings"] = performance_issues
-                # VAMI-013 / ADR-4: parallel warnings_detail (see redis site).
-                performance_metrics["warnings_detail"] = _build_warnings_detail(performance_issues)
-
-            return performance_metrics
-
-        except ImportError:
-            # psutil not available, return basic metrics.
-            # The warning below matches INFO_WARNING_MATCHERS so severity=info.
-            _psutil_warnings = ["psutil not available for detailed metrics"]
-            return {
-                "system": {"status": "metrics_unavailable"},
-                "service": {
-                    "uptime_seconds": (datetime.now(UTC) - self._start_time).total_seconds()
-                },
-                "warnings": _psutil_warnings,
-                "warnings_detail": _build_warnings_detail(_psutil_warnings),
-            }
-        except Exception as e:
-            logger.exception("Error checking performance metrics")
-            return {
-                "system": {"status": "error", "error": str(e)},
-                "service": {
-                    "uptime_seconds": (datetime.now(UTC) - self._start_time).total_seconds()
-                },
-            }
-
-    async def _get_cache_metrics(self) -> dict[str, Any]:
-        """Collect cache hit/miss and lock contention metrics.
-
-        Phase 212: returns a ``status`` field so the cache layer no longer
-        reports ``unknown`` in /api/v1/health/full. Status reflects only
-        whether the redis-analytics counter reads succeeded — not whether
-        the underlying caches themselves are healthy (those live in the
-        redis layer report). Values: ``healthy`` (both reads succeeded),
-        ``degraded`` (one failed), ``unhealthy`` (both failed).
-        """
-        metrics: dict[str, Any] = {
-            "status": "unknown",
-            "search_cache_misses": 0,
-            "analytics_lock_contention": 0,
-            "mem0_latency_samples": 0,
-            "mem0_latency_total_ms": 0,
-            "mem0_search_requests": 0,
-            "mem0_rerank_requests": 0,
-            "mem0_overfetch_count": 0,
-            "mem0_fallback_requests": 0,
-            "mem0_fallback_pgvector": 0,
-            "mem0_fallback_bm25": 0,
-        }
-        analytics_db = getattr(self.config.database, "redis_analytics_db", REDIS_ANALYTICS_DB)
-        primary_ok = False
-        contention_ok = False
-        try:
-            async with redis_manager.get_redis(analytics_db) as conn:
-                misses = await conn.get("metrics:cache:misses")
-                mem0_latency_samples = await conn.get("metric:mem0_latency_samples")
-                mem0_latency_total_ms = await conn.get("metric:mem0_latency_ms_total")
-                mem0_search_requests = await conn.get("metric:mem0_search_requests")
-                mem0_rerank_requests = await conn.get("metric:mem0_rerank_requests")
-                mem0_overfetch_count = await conn.get("metric:mem0_overfetch_count")
-                mem0_fallback_requests = await conn.get("metric:mem0_fallback_requests")
-                mem0_fallback_pgvector = await conn.get("metric:mem0_fallback_pgvector")
-                mem0_fallback_bm25 = await conn.get("metric:mem0_fallback_bm25")
-                metrics["search_cache_misses"] = int(misses or 0)
-                metrics["mem0_latency_samples"] = int(mem0_latency_samples or 0)
-                metrics["mem0_latency_total_ms"] = int(mem0_latency_total_ms or 0)
-                metrics["mem0_search_requests"] = int(mem0_search_requests or 0)
-                metrics["mem0_rerank_requests"] = int(mem0_rerank_requests or 0)
-                metrics["mem0_overfetch_count"] = int(mem0_overfetch_count or 0)
-                metrics["mem0_fallback_requests"] = int(mem0_fallback_requests or 0)
-                metrics["mem0_fallback_pgvector"] = int(mem0_fallback_pgvector or 0)
-                metrics["mem0_fallback_bm25"] = int(mem0_fallback_bm25 or 0)
-                primary_ok = True
-        except Exception as exc:
-            logger.debug("Unable to fetch cache hit/miss metrics: %s", exc)
-
-        try:
-            async with redis_manager.get_redis(analytics_db) as conn:
-                contention = await conn.get("metrics:analytics:lock_contention")
-                metrics["analytics_lock_contention"] = int(contention or 0)
-                contention_ok = True
-        except Exception as exc:
-            logger.debug("Unable to fetch analytics lock contention: %s", exc)
-
-        if primary_ok and contention_ok:
-            metrics["status"] = "healthy"
-        elif primary_ok or contention_ok:
-            metrics["status"] = "degraded"
-        else:
-            metrics["status"] = "unhealthy"
-
-        return metrics
-
-    def _determine_overall_status(
-        self,
-        redis_health: dict[str, Any],
-        postgres_health: dict[str, Any],
-        mem0ai_health: dict[str, Any],
-        memory_evolution_health: dict[str, Any],
-        performance_metrics: dict[str, Any],
-        dedup_health: dict[str, Any] | None = None,
-        embedding_health: dict[str, Any] | None = None,
-    ) -> str:
-        """Determine overall service health status.
-
-        Severity-aware (VAMI-013 / ADR-5): ``info``-tier warnings do NOT drive
-        ``degraded``. Ordering: ``error`` > ``critical`` > ``warning`` > ``info``.
-
-        - any layer ``status == "error"``            → ``unhealthy``
-        - any layer ``status == "degraded"``         → ``degraded``
-        - any warning with severity ∈ {warning, critical} → ``degraded``
-        - else                                         → ``healthy``
-        """
-        statuses = [
-            redis_health.get("status", "unknown"),
-            postgres_health.get("status", "unknown"),
-            mem0ai_health.get("status", "unknown"),
-            memory_evolution_health.get("status", "unknown"),
-        ]
-        if dedup_health:
-            statuses.append(dedup_health.get("status", "unknown"))
-        if embedding_health:
-            statuses.append(embedding_health.get("status", "unknown"))
-
-        # Short-circuit unhealthy on any layer error (FR-13).
-        if any(status == "error" for status in statuses):
-            return "unhealthy"
-
-        # A layer that already classified itself as degraded propagates
-        # directly — its own aggregation logic has already filtered noise.
-        if any(status == "degraded" for status in statuses):
-            return "degraded"
-
-        # Collect all warning entries across layers and performance metrics
-        # and classify each (FR-12). Only warning/critical tiers promote.
-        all_warnings: list[WarningEntry] = []
-        for layer in (
-            redis_health,
-            postgres_health,
-            mem0ai_health,
-            memory_evolution_health,
-        ):
-            all_warnings.extend(layer.get("warnings") or [])
-        if dedup_health:
-            all_warnings.extend(dedup_health.get("warnings") or [])
-        all_warnings.extend(performance_metrics.get("warnings") or [])
-
-        for entry in all_warnings:
-            severity = _classify_warning(entry)
-            if severity in ("warning", "critical"):
-                return "degraded"
-
-        return "healthy"
-
-    def _generate_alerts(self, health_report: dict[str, Any]) -> list[dict[str, Any]]:
-        """Generate alerts for any health issues.
-
-        Severity-aware (VAMI-013 / FR-17): each warning entry is classified via
-        :func:`_classify_warning`; the resulting ``level`` mirrors the severity
-        tier (``info | warning | critical``). Layer-level ``status=error`` is
-        always emitted at ``level=critical``.
-        """
-        alerts = []
-
-        # Check layer-specific issues
-        for layer_name, layer_health in health_report.get("layers", {}).items():
-            if layer_health.get("status") == "error":
-                alerts.append(
-                    {
-                        "level": "critical",
-                        "layer": layer_name,
-                        "message": f"{layer_name.title()} layer error: {layer_health.get('error', 'Unknown error')}",
-                        "timestamp": datetime.now(UTC).isoformat(),
-                    }
-                )
-            elif layer_health.get("warnings"):
-                for warning in layer_health["warnings"]:
-                    alerts.append(
-                        {
-                            "level": _classify_warning(warning),
-                            "layer": layer_name,
-                            "message": f"{layer_name.title()} warning: {_warning_message(warning)}",
-                            "timestamp": datetime.now(UTC).isoformat(),
-                        }
-                    )
-
-        # Check performance issues
-        performance = health_report.get("performance", {})
-        if "warnings" in performance:
-            for warning in performance["warnings"]:
-                alerts.append(
-                    {
-                        "level": _classify_warning(warning),
-                        "layer": "performance",
-                        "message": f"Performance warning: {_warning_message(warning)}",
-                        "timestamp": datetime.now(UTC).isoformat(),
-                    }
-                )
-
-        # Explicit DLQ alert for memory evolution pipeline
-        mem_evo = health_report.get("layers", {}).get("memory_evolution", {})
-        if isinstance(mem_evo, dict):
-            dlq_count = mem_evo.get("dlq_count")
-            if isinstance(dlq_count, int) and dlq_count > EVOLUTION_SLO["dlq_max"]:
-                alerts.append(
-                    {
-                        "level": "warning",
-                        "layer": "memory_evolution",
-                        "message": f"Memory evolution DLQ backlog: {dlq_count}",
-                        "timestamp": datetime.now(UTC).isoformat(),
-                    }
-                )
-
-        return alerts
-
-    async def quick_health_check(self) -> dict[str, Any]:
-        """
-        Quick health check for lightweight monitoring.
-        Returns minimal status for frequent health checks.
-        """
-        try:
-            # Check Redis connectivity only (fastest check)
-            redis_service = await get_redis_service()
-            redis_health = await redis_service.health_check()
-
-            overall_status = "healthy" if redis_health.get("status") == "healthy" else "unhealthy"
-
-            return {
-                "status": overall_status,
-                "timestamp": datetime.now(UTC).isoformat(),
-                "redis_status": redis_health.get("status", "unknown"),
-                "uptime_seconds": (datetime.now(UTC) - self._start_time).total_seconds(),
-            }
-
-        except Exception as e:
-            return {
-                "status": "unhealthy",
-                "timestamp": datetime.now(UTC).isoformat(),
-                "error": str(e),
-            }
-
-
-# Global health check service instance
-health_check_service = HealthCheckService()
-
-
-async def get_health_check_service() -> HealthCheckService:
-    """Get the global health check service instance."""
-    return health_check_service
-
-
-# FastAPI dependency
-async def get_health_checker() -> HealthCheckService:
-    """FastAPI dependency for health check service."""
-    return await get_health_check_service()