@ngocsangairvds/vsaf 3.2.16 → 4.0.1

package/tools/vds-scripts/memory_orchestrator/src/vds_memory_orchestrator/service/main.py

@@ -1,491 +0,0 @@
-"""
-VDS AI Memory Service Main Application
-Advanced AI-powered Memory and Session Management Service.
-"""
-
-import asyncio
-import logging
-import warnings
-from contextlib import asynccontextmanager
-
-import uvicorn
-from fastapi import FastAPI, Request, status
-from fastapi.middleware.cors import CORSMiddleware
-from fastapi.middleware.trustedhost import TrustedHostMiddleware
-from fastapi.responses import JSONResponse
-
-# Load environment variables explicitly
-try:
-    from dotenv import load_dotenv
-
-    load_dotenv()
-    logging.info("✅ Environment variables loaded from .env file")
-except ImportError:
-    logging.warning("⚠️ python-dotenv not available, environment variables may not be loaded")
-except Exception as e:
-    logging.warning(f"⚠️ Failed to load .env file: {e}")
-
-# Configure logging — FR-6.3: platform standard structured JSON logging
-from vds_platform_core.logging import configure_structlog
-
-from .api.ai_memory_api import router as ai_memory_router
-from .api.mcp_api import router as mcp_router
-
-# VAMI-028 FR-28.1 (ADR-43 Option B): api/v1/search.py + services/memory_search.py
-# deleted as orphan code (copied from archived vds-ai-memory in VAMI-010 M2,
-# untouched since; live search uses ai_memory_api.py + memory_search_service.py).
-from .api.v1.analytics import router as analytics_router
-from .api.v1.auth import router as auth_router
-from .api.v1.authorization import router as authorization_router
-from .api.v1.cache import router as cache_router
-from .api.v1.cleanup import router as cleanup_router
-
-# Temporarily disabled due to ML dependency compatibility issues
-from .api.v1.deduplication import router as deduplication_router
-from .api.v1.embedding import router as embedding_router
-from .api.v1.export_import import router as export_import_router
-from .api.v1.health import router as health_router  # Phase 229 FR-229.4
-from .api.v1.hooks import router as hooks_router
-from .api.v1.maintenance import router as maintenance_router  # Phase 201 ADR-65
-from .api.v1.promotion import router as promotion_router
-from .api.v1.reconciliation import router as reconciliation_router
-from .api.v1.sharing import router as sharing_router
-from .api.v1.tasks import router as tasks_router
-from .api.v1.versioning import router as versioning_router
-from .core.ai_memory_exceptions import (
-    ConfigurationError,
-    NotFoundError,
-    RateLimitError,
-    ServiceUnavailableError,
-    ValidationError,
-)
-from .core.config import get_config
-from .database.redis_manager import redis_service
-
-configure_structlog(json_logs=True)
-logger = logging.getLogger(__name__)
-
-# Silence upstream asyncio DeprecationWarning emitted by backoff until library update
-warnings.filterwarnings(
-    "ignore",
-    message=r".*asyncio\\.iscoroutinefunction.*",
-    category=DeprecationWarning,
-    module=r"backoff\\._decorator",
-)
-
-from .mcp.official_mcp_client import official_mcp_client
-from .services.ai_memory_service import get_ai_memory_service
-from .services.analytics_events import record_response_metrics
-
-# Global configuration
-config = get_config()
-
-
-@asynccontextmanager
-async def lifespan(_app: FastAPI):
-    """Application lifespan manager."""
-    # Startup
-    logger.info("Starting VDS Unified Service...")
-
-    try:
-        # Initialize Redis service
-        await redis_service.initialize()
-        logger.info("✅ Redis service initialized")
-
-        # Initialize database service
-        from .database.postgres_manager import initialize_database
-
-        try:
-            await initialize_database()
-            logger.info("✅ Database service initialized")
-        except Exception as db_error:
-            logger.warning(f"⚠️ Database initialization failed: {db_error}")
-            logger.info("🔄 Continuing without database storage")
-
-        # Initialize AI memory service
-        ai_memory_service = await get_ai_memory_service()
-        await ai_memory_service.initialize()
-        logger.info("✅ AI memory service initialized")
-
-        # Initialize MCP client if enabled
-        if config.mcp.enable_mcp_server:
-            try:
-                mcp_config_file = (
-                    "mcp_config.yaml" if config.is_development else "/etc/vds/mcp_config.yaml"
-                )
-                await official_mcp_client.initialize(mcp_config_file)
-                logger.info("✅ MCP client initialized")
-            except Exception as mcp_error:
-                logger.warning(f"⚠️ MCP client initialization failed: {mcp_error}")
-                logger.info("🔄 Continuing without MCP integration")
-
-        # Start background tasks
-        await ai_memory_service._start_background_tasks()
-        logger.info("✅ Background tasks started")
-
-        # Health check
-        redis_health = await redis_service.health_check()
-        logger.info("✅ Redis health: %s", redis_health.get("status", "unknown"))
-
-        logger.info("🚀 VDS AI Memory Service started successfully")
-
-        yield
-
-    except Exception as e:
-        logger.exception(f"❌ Failed to start VDS AI Memory Service: {e}")
-        raise
-
-    # Shutdown
-    logger.info("Shutting down VDS AI Memory Service...")
-
-    try:
-        # Shutdown MCP client if initialized
-        try:
-            await official_mcp_client.shutdown()
-            logger.info("✅ MCP client shutdown complete")
-        except Exception as mcp_error:
-            logger.warning(f"Warning during MCP client shutdown: {mcp_error}")
-
-        # Get AI memory service and perform graceful shutdown
-        try:
-            ai_memory_service = await get_ai_memory_service()
-            await ai_memory_service.shutdown()
-            logger.info("✅ Service shutdown complete")
-        except Exception as e:
-            logger.warning(f"Warning during service shutdown: {e}")
-
-        logger.info("✅ VDS AI Memory Service shut down successfully")
-
-    except Exception as e:
-        logger.exception(f"❌ Error during shutdown: {e}")
-
-
-# Create FastAPI application
-app = FastAPI(
-    title="VDS AI Memory Service",
-    description="""
-    ## Viettel Digital Services AI Memory Platform
-
-    Advanced AI-powered Memory and Session management service with cognitive capabilities.
-
-    ### Features
-    - **Memory Management**: Advanced memory storage, search, and retrieval with Mem0AI integration
-    - **Session Management**: Comprehensive session lifecycle management with Redis persistence
-    - **Analytics**: Real-time analytics and insights generation
-    - **Graph Memory**: Advanced relationship mapping and knowledge graphs
-    - **High Performance**: Redis-backed caching and async operations
-    - **Enterprise Ready**: Comprehensive monitoring, logging, and error handling
-
-    ### Architecture
-    - **FastAPI**: Modern async web framework with automatic OpenAPI documentation
-    - **Redis**: Multi-database architecture (memory cache, sessions, analytics)
-    - **Mem0AI**: Advanced AI memory platform with Pro features
-    - **Pydantic**: Data validation and serialization with V2
-
-    ### API Documentation
-    - **Swagger UI**: Available at `/docs` for interactive API exploration
-    - **OpenAPI**: Available at `/openapi.json` for API specification
-    - **Health Check**: Available at `/api/v1/health` for service status
-
-    ### Business Domains Supported
-    - **LEP**: Lending Platform (25+ microservices)
-    - **INSURANCE**: Insurance Platform (22+ microservices)
-    - **EKYC**: Electronic KYC Services
-    - **PLATFORM**: Shared Infrastructure Services
-
-    ### Pro Features
-    - Unlimited memory storage and advanced analytics
-    - Graph memory with relationship mapping
-    - Enhanced reranking and search capabilities
-    - Team collaboration and custom models
-    """,
-    version="2.0.0",
-    lifespan=lifespan,
-    docs_url="/docs",
-    redoc_url="/redoc",
-    openapi_url="/openapi.json",
-)
-
-# Add CORS middleware
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=config.service.cors_origins,
-    allow_credentials=True,
-    allow_methods=["*"],
-    allow_headers=["*"],
-)
-
-# Add trusted host middleware
-app.add_middleware(
-    TrustedHostMiddleware,
-    allowed_hosts=["*"],  # Configure based on your deployment needs
-)
-
-# Add security headers middleware (CSP, X-Content-Type-Options, etc.)
-from .core.security_middleware import SecurityHeadersMiddleware
-
-app.add_middleware(SecurityHeadersMiddleware)
-
-
-@app.middleware("http")
-async def logging_middleware(request: Request, call_next):
-    """Request logging and metrics middleware."""
-    start_time = asyncio.get_event_loop().time()
-
-    # Log request
-    logger.info(
-        f"📥 {request.method} {request.url.path} - Client: {request.client.host if request.client else 'unknown'}"
-    )
-
-    try:
-        response = await call_next(request)
-    except NotFoundError as exc:
-        process_time = asyncio.get_event_loop().time() - start_time
-        process_time_ms = process_time * 1000
-        try:
-            await record_response_metrics(process_time_ms, 404)
-        except Exception as metrics_error:
-            logger.debug(f"Failed to record error metrics: {metrics_error}")
-        logger.info(
-            f"ℹ️ {request.method} {request.url.path} - Not found: {exc!s} - Time: {process_time:.3f}s"
-        )
-        raise
-    except ValidationError as exc:
-        process_time = asyncio.get_event_loop().time() - start_time
-        process_time_ms = process_time * 1000
-        try:
-            await record_response_metrics(process_time_ms, 422)
-        except Exception as metrics_error:
-            logger.debug(f"Failed to record error metrics: {metrics_error}")
-        logger.warning(
-            f"⚠️ {request.method} {request.url.path} - Validation error: {exc!s} - Time: {process_time:.3f}s"
-        )
-        raise
-    except RateLimitError as exc:
-        process_time = asyncio.get_event_loop().time() - start_time
-        process_time_ms = process_time * 1000
-        try:
-            await record_response_metrics(process_time_ms, 429)
-        except Exception as metrics_error:
-            logger.debug(f"Failed to record error metrics: {metrics_error}")
-        logger.warning(
-            f"⚠️ {request.method} {request.url.path} - Rate limit: {exc!s} - Time: {process_time:.3f}s"
-        )
-        raise
-    except Exception as exc:
-        process_time = asyncio.get_event_loop().time() - start_time
-        process_time_ms = process_time * 1000
-        try:
-            await record_response_metrics(process_time_ms, 500)
-        except Exception as metrics_error:
-            logger.debug(f"Failed to record error metrics: {metrics_error}")
-        logger.exception(
-            f"❌ {request.method} {request.url.path} - Error: {exc!s} - Time: {process_time:.3f}s"
-        )
-        raise
-
-    process_time = asyncio.get_event_loop().time() - start_time
-    process_time_ms = process_time * 1000
-
-    try:
-        await record_response_metrics(process_time_ms, response.status_code)
-    except Exception as metrics_error:
-        logger.debug(f"Failed to track metrics: {metrics_error}")
-
-    # Log response
-    logger.info(
-        f"📤 {request.method} {request.url.path} - Status: {response.status_code} - Time: {process_time:.3f}s"
-    )
-
-    # Add processing time header
-    response.headers["X-Process-Time"] = str(process_time)
-
-    return response
-
-
-@app.middleware("http")
-async def error_handling_middleware(request: Request, call_next):
-    """Global error handling middleware."""
-    try:
-        return await call_next(request)
-
-    except ValidationError as e:
-        logger.warning(f"Validation error for {request.url.path}: {e}")
-        return JSONResponse(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            content={
-                "error": "Validation Error",
-                "message": str(e),
-                "type": "validation_error",
-                "path": request.url.path,
-            },
-        )
-
-    except NotFoundError as e:
-        logger.warning(f"Not found error for {request.url.path}: {e}")
-        return JSONResponse(
-            status_code=status.HTTP_404_NOT_FOUND,
-            content={
-                "error": "Not Found",
-                "message": str(e),
-                "type": "not_found_error",
-                "path": request.url.path,
-            },
-        )
-
-    except RateLimitError as e:
-        logger.warning(f"Rate limit error for {request.url.path}: {e}")
-        return JSONResponse(
-            status_code=status.HTTP_429_TOO_MANY_REQUESTS,
-            content={
-                "error": "Rate Limit Exceeded",
-                "message": str(e),
-                "type": "rate_limit_error",
-                "path": request.url.path,
-            },
-        )
-
-    except ServiceUnavailableError as e:
-        logger.exception(f"Service unavailable error for {request.url.path}: {e}")
-        return JSONResponse(
-            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
-            content={
-                "error": "Service Unavailable",
-                "message": str(e),
-                "type": "service_unavailable_error",
-                "path": request.url.path,
-            },
-        )
-
-    except ConfigurationError as e:
-        logger.exception(f"Configuration error for {request.url.path}: {e}")
-        return JSONResponse(
-            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            content={
-                "error": "Configuration Error",
-                "message": "Service configuration error",
-                "type": "configuration_error",
-                "path": request.url.path,
-            },
-        )
-
-    except Exception as e:
-        logger.error(f"Unexpected error for {request.url.path}: {e}", exc_info=True)
-        return JSONResponse(
-            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            content={
-                "error": "Internal Server Error",
-                "message": "An unexpected error occurred",
-                "type": "internal_server_error",
-                "path": request.url.path,
-            },
-        )
-
-
-# Include API routers
-app.include_router(ai_memory_router, tags=["ai_memory"])
-app.include_router(export_import_router, tags=["export_import"])
-app.include_router(versioning_router, tags=["versioning"])
-app.include_router(sharing_router, tags=["sharing"])
-app.include_router(auth_router, tags=["authentication"])
-app.include_router(authorization_router, tags=["authorization"])
-# Temporarily disabled due to ML dependency compatibility issues
-app.include_router(deduplication_router, tags=["deduplication"])
-# VAMI-028 FR-28.1: search_router removed (orphan deletion)
-app.include_router(analytics_router, tags=["analytics"])
-app.include_router(reconciliation_router, tags=["reconciliation"])
-app.include_router(cleanup_router, tags=["cleanup"])
-app.include_router(tasks_router, tags=["tasks"])
-app.include_router(cache_router, tags=["cache"])
-app.include_router(embedding_router, tags=["embedding"])
-app.include_router(hooks_router, tags=["hooks"])
-app.include_router(promotion_router, tags=["promotion"])
-app.include_router(health_router, tags=["health"])  # Phase 229 FR-229.4
-app.include_router(maintenance_router, tags=["maintenance"])  # Phase 201 ADR-65
-
-# Include MCP router if enabled
-if config.mcp.enable_mcp_server:
-    app.include_router(mcp_router, tags=["mcp"])
-
-
-# Root endpoint
-@app.get("/", tags=["root"])
-async def root():
-    """Root endpoint with service information."""
-    return {
-        "service": {
-            "name": "VDS AI Memory Service",
-            "version": "2.0.0",
-            "description": "Advanced AI-powered Memory and Session Management Service",
-            "status": "operational",
-        },
-        "features": {
-            "memory_management": True,
-            "session_management": True,
-            "analytics": True,
-            "pro_features": config.mem0.api_key != "",
-            "mcp_server": config.mcp.enable_mcp_server,
-        },
-        "endpoints": {
-            "api_docs": "/docs",
-            "health_check": "/api/v1/health",
-            "configuration": "/api/v1/config",
-            "statistics": "/api/v1/stats",
-        },
-        "business_domains": ["LEP", "INSURANCE", "EKYC", "PLATFORM"],
-        "technology": {
-            "framework": "FastAPI",
-            "python_version": "3.14+",
-            "cache": "Redis",
-            "ai_platform": "Mem0AI Pro",
-            "validation": "Pydantic V2",
-        },
-    }
-
-
-# Custom OpenAPI schema with enhanced documentation
-def custom_openapi():
-    from .api.openapi_config import get_custom_openapi_schema
-
-    return get_custom_openapi_schema(app)
-
-
-app.openapi = custom_openapi
-
-
-# Health check endpoint (quick access)
-@app.get("/health", tags=["health"])
-async def quick_health_check():
-    """Quick health check endpoint."""
-    try:
-        # Quick/cheap health: avoid heavy Mem0 checks; just ensure process is alive
-        from datetime import UTC, datetime
-
-        return {
-            "status": {"overall": "healthy"},
-            "service_name": "VDS AI Memory Service",
-            "timestamp": datetime.now(UTC).isoformat(),
-            "components": {"api": "ok"},
-        }
-
-    except Exception as e:
-        logger.exception(f"Health check failed: {e}")
-        return JSONResponse(
-            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
-            content={
-                "status": "unhealthy",
-                "error": str(e),
-                "timestamp": asyncio.get_event_loop().time(),
-            },
-        )
-
-
-# Development server configuration
-if __name__ == "__main__":
-    uvicorn.run(
-        "app.main:app",
-        host="0.0.0.0",
-        port=8000,
-        reload=config.service.debug,
-        log_level="info",
-    )

package/tools/vds-scripts/memory_orchestrator/src/vds_memory_orchestrator/service/mcp/__init__.py

@@ -1,20 +0,0 @@
-"""
-MCP (Model Context Protocol) Client Infrastructure
-
-This module provides the core MCP client components for VDS-AI-Memory service,
-enabling integration with external MCP servers and exposing VDS capabilities as MCP resources.
-"""
-
-__version__ = "1.0.0"
-__author__ = "VDS Development Team"
-
-from .official_mcp_client import get_mcp_client, official_mcp_client
-
-# Backward compatibility alias expected by legacy tests
-client = official_mcp_client
-
-__all__ = [
-    "official_mcp_client",
-    "get_mcp_client",
-    "client",
-]

package/tools/vds-scripts/memory_orchestrator/src/vds_memory_orchestrator/service/mcp/env_config_parser.py

@@ -1,188 +0,0 @@
-"""
-MCP Server Configuration Parser from Environment Variables.
-
-This module provides a data-driven way to configure external MCP servers
-by parsing environment variables with a specific prefix (`MCP_SERVER_`).
-It uses Pydantic for validation and type casting.
-"""
-
-import json
-import logging
-import os
-from collections import defaultdict
-
-from pydantic import BaseModel, Field, field_validator
-
-logger = logging.getLogger(__name__)
-
-# A mapping of known server names to their capabilities.
-# This can be extended as new standard servers are integrated.
-# The environment variable `MCP_SERVER_<NAME>_CAPABILITIES` will always override this.
-KNOWN_SERVER_CAPABILITIES = {
-    "EXA": ["web_search_exa", "get_code_context_exa"],
-    "TAVILY": ["tavily_search", "tavily_extract", "tavily_crawl", "tavily_map"],
-    "CONTEXT7": ["resolve_library_id", "get_library_docs"],
-    "DEEPWIKI": ["read_wiki_structure", "read_wiki_contents", "ask_question"],
-    "BRIGHTDATA": ["search_engine", "scrape_batch", "scrape_as_markdown"],
-    "BRAVE": [
-        "brave_web_search",
-        "brave_image_search",
-        "brave_news_search",
-        "brave_video_search",
-    ],
-    "PERPLEXITY": [
-        "perplexity_search",
-        "perplexity_reason",
-        "perplexity_deep_research",
-    ],
-    "ZEN": ["zen_models", "zen_chat", "zen_thinkdeep", "zen_codereview", "zen_analyze"],
-}
-
-
-class MCPServerEnvConfig(BaseModel):
-    """
-    Pydantic model for a single MCP server's configuration from environment variables.
-    """
-
-    name: str
-    enabled: bool = True
-    transport: str = "http"
-    url: str | None = None
-    api_key: str | None = None
-    capabilities: list[str] = Field(default_factory=list)
-    env: dict[str, str] = Field(default_factory=dict)
-    args: list[str] = Field(default_factory=list)
-    command: str | None = None
-    timeout: float = 30.0
-    max_connections: int = 5
-
-    @field_validator("capabilities", "args", mode="before")
-    def split_str(cls, v):
-        if isinstance(v, str):
-            return [item.strip() for item in v.split(",") if item.strip()]
-        return v
-
-    @field_validator("env", mode="before")
-    def parse_json_dict(cls, v):
-        if isinstance(v, str):
-            try:
-                return json.loads(v)
-            except json.JSONDecodeError:
-                raise ValueError("Invalid JSON format for 'env' field")
-        return v
-
-    # Simplified validation - let the application logic handle URL validation
-    # This allows more flexible configuration while still providing structure
-
-
-def load_mcp_servers_from_env() -> list[MCPServerEnvConfig]:
-    """
-    Scans environment variables for MCP server configurations and parses them.
-
-    Looks for variables matching the pattern `MCP_SERVER_<NAME>_<ATTRIBUTE>`.
-
-    Returns:
-        A list of validated MCPServerEnvConfig objects.
-
-    Raises:
-        ConfigurationError: If any server configuration is invalid.
-    """
-    prefix = "MCP_SERVER_"
-    server_configs_raw = defaultdict(dict)
-
-    for key, value in os.environ.items():
-        if key.startswith(prefix):
-            try:
-                # Remove the prefix and split the remaining part
-                remaining = key[len(prefix) :]  # Remove "MCP_SERVER_"
-
-                # Find the last underscore to separate server name from attribute
-                if "_" not in remaining:
-                    logger.debug(f"Skipping malformed MCP variable (no underscore): {key}")
-                    continue
-
-                # List of known compound attributes that can appear after server names
-                COMPOUND_ATTRIBUTES = {
-                    "MAX_CONNECTIONS",
-                    "API_KEY",
-                    "TIMEOUT",
-                    "START_PERIOD",
-                    "HEALTH_CHECK_INTERVAL",
-                    "SOCKET_TIMEOUT",
-                    "SOCKET_CONNECT_TIMEOUT",
-                }
-
-                # Try to match compound attributes first
-                attr_found = None
-                server_name = None
-
-                for compound_attr in sorted(COMPOUND_ATTRIBUTES, key=len, reverse=True):
-                    logger.debug(
-                        f"Checking compound attribute: '{compound_attr}' against remaining: '{remaining}'"
-                    )
-                    if remaining.endswith(compound_attr):
-                        # Check if there's an underscore before the compound attribute
-                        if (
-                            len(remaining) > len(compound_attr)
-                            and remaining[-len(compound_attr) - 1] == "_"
-                        ):
-                            attr_found = compound_attr.lower()
-                            server_name = remaining[
-                                : -(len(compound_attr) + 1)
-                            ]  # Remove trailing _ and compound_attr
-                            logger.debug(
-                                f"Compound attribute match: remaining='{remaining}', compound_attr='{compound_attr}', server_name='{server_name}'"
-                            )
-                            break
-
-                # If no compound attribute found, use regular parsing
-                if attr_found is None:
-                    # Split on the last underscore to handle server names with underscores
-                    last_underscore = remaining.rfind("_")
-                    server_name = remaining[:last_underscore]
-                    attr_found = remaining[last_underscore + 1 :].lower()
-
-                attr = attr_found
-                server_configs_raw[server_name][attr] = value
-                logger.debug(f"Parsed: server='{server_name}', attr='{attr}', value='{value}'")
-            except ValueError:
-                logger.warning(f"Malformed MCP server environment variable ignored: {key}")
-
-    validated_servers = []
-    for name, config_dict in server_configs_raw.items():
-        # Use a default set of capabilities if not provided in env
-        if "capabilities" not in config_dict:
-            default_caps = KNOWN_SERVER_CAPABILITIES.get(name)
-            if default_caps:
-                config_dict["capabilities"] = default_caps
-                logger.debug(f"Using default capabilities for MCP server {name}")
-
-        try:
-            config = MCPServerEnvConfig(name=name.lower().replace("_", "-"), **config_dict)
-            if not config.enabled:
-                logger.info(
-                    f"MCP server '{config.name}' is disabled via environment configuration."
-                )
-                continue
-
-            if config.url and "${" in config.url:
-                logger.warning(
-                    "Skipping MCP server '%s': URL contains unresolved placeholder (%s)",
-                    config.name,
-                    config.url,
-                )
-                continue
-
-            # Basic validation for API keys
-            if config.api_key and config.api_key.startswith(("your-", "placeholder")):
-                logger.warning(f"Skipping MCP server '{config.name}': API key is a placeholder.")
-                continue
-
-            validated_servers.append(config)
-        except Exception:
-            raise
-
-    logger.info(
-        f"Found and validated {len(validated_servers)} enabled MCP servers from environment variables."
-    )
-    return validated_servers