@ngocsangairvds/vsaf 3.2.16 → 4.0.0

package/tools/vds-scripts/memory_orchestrator/tests/service/test_circuit_breaker_integration.py

@@ -1,336 +0,0 @@
-"""
-Circuit breaker integration tests for VDS AI Memory.
-
-Tests the embedded circuit breaker implementation in ai_memory_service.py:
-- State transitions: CLOSED → OPEN → HALF_OPEN → CLOSED
-- Failure threshold: 5 consecutive failures
-- Reset timeout: 60 seconds
-- Health metrics exposure
-
-Sprint 2 Reliability Feature Validation.
-"""
-
-import time
-from typing import Any
-
-import httpx
-import pytest
-
-
-async def get_circuit_breaker_state(api_client: httpx.AsyncClient) -> dict[str, Any]:
-    """Get current circuit breaker state from health endpoint."""
-    response = await api_client.get("/api/v1/health/full")
-    assert response.status_code == 200
-    health = response.json()
-    return health["layers"]["mem0ai"].get("circuit_breaker", {})
-
-
-@pytest.mark.asyncio
-@pytest.mark.load
-async def test_circuit_breaker_closed_state_initially(api_client: httpx.AsyncClient):
-    """
-    Verify circuit breaker starts in CLOSED state (healthy service).
-
-    Expected: state=closed, failure_count=0
-    """
-    state = await get_circuit_breaker_state(api_client)
-
-    print("\n🔌 Initial Circuit Breaker State:")
-    print(f"   - state: {state.get('state')}")
-    print(f"   - failures: {state.get('failures')}")  # Fixed field name
-    print(f"   - failure_threshold: {state.get('failure_threshold')}")  # Fixed field name
-    print(f"   - reset_seconds: {state.get('reset_seconds')}")
-
-    # Verify initial state is CLOSED (healthy)
-    assert state.get("state") in ["closed", "unknown"], (
-        "Circuit breaker should start in CLOSED or unknown state"
-    )
-    assert state.get("failures", 0) >= 0, "Failure count should be non-negative"
-    assert state.get("failure_threshold", 5) == 5, "Default threshold should be 5 failures"
-
-
-@pytest.mark.asyncio
-@pytest.mark.load
-async def test_circuit_breaker_opens_after_failures(api_client: httpx.AsyncClient):
-    """
-    Test that circuit breaker opens after 5 consecutive Mem0 failures.
-
-    Strategy:
-    1. Record initial state (should be CLOSED)
-    2. Simulate 5 failures by searching with invalid API key
-    3. Verify state transitions to OPEN
-    4. Verify service raises 503 during OPEN state
-    """
-    # Step 1: Get initial state
-    initial_state = await get_circuit_breaker_state(api_client)
-    assert initial_state.get("state") == "closed", "Circuit breaker should start CLOSED"
-
-    print("\n🔌 Testing OPEN state transition:")
-    print(
-        f"   Initial state: {initial_state.get('state')}, failures: {initial_state.get('failures')}"
-    )
-
-    # Step 2: Simulate 5 consecutive Mem0 failures
-    # We'll do this by creating memories with force_mem0=true
-    # which will fail if there are any Mem0 API issues
-
-    # First, let's check the current health to see if Mem0 is accessible
-    health_response = await api_client.get("/api/v1/health/full")
-    health = health_response.json()
-    mem0_status = health["layers"]["mem0ai"].get("status", "unknown")
-
-    print(f"   Mem0 status: {mem0_status}")
-
-    # If Mem0 is already degraded/unavailable, we can observe the circuit state
-    if mem0_status != "healthy":
-        # Mem0 is already having issues - let's observe the circuit state
-        state_after = await get_circuit_breaker_state(api_client)
-        print(f"   Circuit state with unhealthy Mem0: {state_after.get('state')}")
-        print(f"   Failure count: {state_after.get('failures')}")
-
-        # Verify circuit is either OPEN or HALF_OPEN
-        assert state_after.get("state") in ["open", "half_open"], (
-            f"Expected OPEN or HALF_OPEN, got {state_after.get('state')}"
-        )
-
-        print(
-            f"   ✅ Circuit breaker is {state_after.get('state').upper()} (as expected with degraded Mem0)"
-        )
-        return
-
-    # If Mem0 is healthy, we need to simulate failures
-    # Since we can't actually break the Mem0 API, we'll verify the circuit breaker
-    # logic by checking the health endpoint shows the correct structure
-
-    # Verify circuit breaker metrics are exposed
-    state = await get_circuit_breaker_state(api_client)
-
-    print("\n📊 Circuit Breaker Metrics Exposed:")
-    print(f"   - state: {state.get('state')}")
-    print(f"   - failures: {state.get('failures')}")
-    print(f"   - failure_threshold: {state.get('failure_threshold')}")
-    print(f"   - reset_seconds: {state.get('reset_seconds')}")
-    print(f"   - last_failure_at: {state.get('last_failure_at')}")
-    print(f"   - reset_at: {state.get('reset_at')}")
-
-    # Verify all required fields are present
-    assert "state" in state, "Circuit breaker state must be exposed"
-    assert "failures" in state or "error" in state, "Failures or error must be present"
-    assert "failure_threshold" in state or "error" in state, "Threshold or error must be present"
-    assert "reset_seconds" in state, "Reset seconds must be exposed"
-
-    print("\n✅ Circuit breaker metrics are properly exposed in health endpoint")
-
-
-@pytest.mark.asyncio
-@pytest.mark.load
-async def test_circuit_breaker_half_open_after_timeout(api_client: httpx.AsyncClient):
-    """
-    Test that circuit breaker transitions to HALF_OPEN after reset timeout.
-
-    Strategy:
-    1. Wait for circuit to be OPEN (5+ failures)
-    2. Wait 60+ seconds for reset timeout
-    3. Verify state transitions to HALF_OPEN
-    4. Verify service allows probe calls in HALF_OPEN state
-
-    Note: This test requires Mem0 to be in degraded state or simulates
-    the timeout scenario by checking the health metrics.
-    """
-    state = await get_circuit_breaker_state(api_client)
-
-    print("\n🔌 Testing HALF_OPEN state transition:")
-    print(f"   Current state: {state.get('state')}")
-    print(f"   Failure count: {state.get('failures')}")
-
-    # If circuit is OPEN, check if we're close to reset timeout
-    if state.get("state") == "open":
-        reset_at = state.get("reset_at")
-        if reset_at:
-            now = time.time()
-            reset_timestamp = reset_at
-
-            # Calculate remaining time
-            remaining = reset_timestamp - now
-
-            print(f"   Reset at: {reset_at}")
-            print(f"   Current time: {now}")
-            print(f"   Remaining seconds until HALF_OPEN: {remaining:.1f}")
-
-            if remaining > 0:
-                print(f"   ⏱️  Circuit is OPEN, will transition to HALF_OPEN in {remaining:.1f}s")
-            else:
-                print("   ✅ Circuit should be in HALF_OPEN (reset time passed)")
-
-    # Verify the state transition logic
-    # The circuit should be:
-    # - CLOSED if failures < threshold
-    # - OPEN if failures >= threshold AND now < reset_at
-    # - HALF_OPEN if failures >= threshold AND now >= reset_at
-
-    threshold = state.get("failure_threshold", 5)
-    failure_count = state.get("failures", 0)
-    reset_at = state.get("reset_at")
-    current_state = state.get("state")
-
-    # Validate state logic
-    if failure_count < threshold:
-        expected_state = "closed"
-    elif reset_at and time.time() < reset_at:
-        expected_state = "open"
-    else:
-        expected_state = "half_open" if failure_count >= threshold else "closed"
-
-    print("\n✅ State validation:")
-    print(f"   Expected: {expected_state}, Actual: {current_state}")
-    print(f"   Match: {expected_state == current_state}")
-
-    assert current_state == expected_state, (
-        f"State mismatch: expected {expected_state}, got {current_state}"
-    )
-
-
-@pytest.mark.asyncio
-@pytest.mark.load
-async def test_circuit_breaker_closes_after_success(api_client: httpx.AsyncClient):
-    """
-    Test that circuit breaker returns to CLOSED after successful call.
-
-    Strategy:
-    1. Get circuit in HALF_OPEN state
-    2. Perform a successful search
-    3. Verify failure_count resets to 0
-    4. Verify state returns to CLOSED
-    """
-    # First, perform a successful search to ensure service is working
-    search_response = await api_client.post(
-        "/api/v1/memories/search",
-        json={
-            "query": "test search for circuit breaker recovery",
-            "user_id": "circuit-breaker-test",
-            "session_id": "test-session",
-            "limit": 5,
-        },
-    )
-
-    # Search should succeed (even if Mem0 fails, fallback should work)
-    assert search_response.status_code in [200, 503], (
-        f"Expected 200 or 503, got {search_response.status_code}"
-    )
-
-    # Get circuit breaker state after successful call
-    state = await get_circuit_breaker_state(api_client)
-
-    print("\n🔌 Testing CLOSED state recovery:")
-    print(f"   Search response status: {search_response.status_code}")
-    print(f"   Circuit state: {state.get('state')}")
-    print(f"   Failure count: {state.get('failure_count')}")
-
-    # If search was successful (200), circuit should be CLOSED or moving toward CLOSED
-    if search_response.status_code == 200:
-        result = search_response.json()
-        search_mode = result.get("search_mode", "")
-
-        print(f"   Search mode: {search_mode}")
-
-        # If search succeeded via Mem0, circuit should be CLOSED
-        if search_mode == "mem0":
-            assert state.get("state") == "closed", (
-                "Circuit should be CLOSED after successful Mem0 call"
-            )
-            print("   ✅ Circuit is CLOSED after successful Mem0 call")
-        else:
-            print(
-                f"   ℹ️  Search succeeded via fallback ({search_mode}), circuit state: {state.get('state')}"
-            )
-    else:
-        print("   ℹ️  Search failed (503), circuit may be OPEN")
-
-    # Verify circuit breaker metrics are tracked (field is "failures" in current API)
-    failure_key = "failures" if "failures" in state else "failure_count"
-    assert failure_key in state, "Failure count must be tracked"
-    assert isinstance(state.get(failure_key), int), "Failure count must be integer"
-
-    print("\n✅ Circuit breaker metrics are properly tracked")
-
-
-@pytest.mark.asyncio
-@pytest.mark.load
-async def test_circuit_breaker_configuration(api_client: httpx.AsyncClient):
-    """
-    Verify circuit breaker configuration is correct.
-
-    Tests:
-    - Threshold: 5 failures
-    - Reset timeout: 60 seconds
-    - State enum: closed, open, half_open
-    """
-    state = await get_circuit_breaker_state(api_client)
-
-    print("\n⚙️  Circuit Breaker Configuration:")
-    print(f"   - failure_threshold: {state.get('failure_threshold')}")
-    print(f"   - reset_seconds: {state.get('reset_seconds')}")
-    print(f"   - state: {state.get('state')}")
-    print(f"   - failures: {state.get('failures')}")
-
-    # Verify configuration
-    assert state.get("failure_threshold") == 5 or state.get("error"), (
-        "Threshold should be 5 failures or error present"
-    )
-    assert state.get("reset_seconds") == 60 or state.get("error"), (
-        "Reset timeout should be 60 seconds or error present"
-    )
-    assert state.get("state") in ["closed", "open", "half_open", "unknown"], (
-        f"Invalid state: {state.get('state')}"
-    )
-
-    print("\n✅ Circuit breaker configuration is correct")
-
-
-@pytest.mark.asyncio
-@pytest.mark.load
-async def test_circuit_breaker_metrics_in_health_endpoint(api_client: httpx.AsyncClient):
-    """
-    Verify circuit breaker metrics are exposed in health endpoint.
-
-    Checks that /api/v1/health/full includes circuit_breaker section
-    with all required fields.
-    """
-    response = await api_client.get("/api/v1/health/full")
-    assert response.status_code == 200
-
-    health = response.json()
-    mem0ai = health["layers"].get("mem0ai", {})
-    circuit_breaker = mem0ai.get("circuit_breaker", {})
-
-    print("\n📊 Circuit Breaker Health Metrics:")
-    print(f"   Available fields: {list(circuit_breaker.keys())}")
-
-    # Verify required fields
-    required_fields = [
-        "state",
-        "failure_count",
-        "threshold",
-        "reset_seconds",
-        "last_failure_at",
-        "reset_at",
-    ]
-
-    missing_fields = [field for field in required_fields if field not in circuit_breaker]
-
-    if missing_fields:
-        print(f"   ⚠️  Missing fields: {missing_fields}")
-    else:
-        print("   ✅ All required fields present")
-
-    # At minimum, state and failure count must be present (field is "failures" in current API)
-    assert "state" in circuit_breaker, "State field is required"
-    failure_key = "failures" if "failures" in circuit_breaker else "failure_count"
-    assert failure_key in circuit_breaker, "Failure count field is required"
-
-    # Verify state is valid
-    assert circuit_breaker["state"] in ["closed", "open", "half_open"], (
-        f"Invalid state: {circuit_breaker['state']}"
-    )
-
-    print("\n✅ Circuit breaker metrics are exposed in health endpoint")

package/tools/vds-scripts/memory_orchestrator/tests/service/test_cleanup_endpoint.py

@@ -1,126 +0,0 @@
-"""Tests for POST /api/v1/cleanup endpoint (TSK-4.11)."""
-
-from unittest.mock import AsyncMock, patch
-
-import pytest
-from httpx import ASGITransport, AsyncClient
-
-from vds_memory_orchestrator.service.main import app
-
-
-@pytest.mark.asyncio
-async def test_cleanup_dry_run_returns_candidates_without_deleting():
-    """
-    Dry-run mode (default) must return the candidate list but perform no writes.
-    Covers: one dry-run test per spec requirement (TSK-4.11).
-    """
-    # Patch at the helper-function level — isolates endpoint logic from DB layer.
-    with (
-        patch(
-            "vds_memory_orchestrator.service.api.v1.cleanup._find_debris_memories",
-            new=AsyncMock(return_value=(["_healthcheck_probe", "_test_e2e"], 4)),
-        ),
-        patch(
-            "vds_memory_orchestrator.service.api.v1.cleanup._delete_debris_memories",
-            new=AsyncMock(return_value=0),
-        ) as mock_delete,
-    ):
-        async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
-            response = await client.post(
-                "/api/v1/cleanup",
-                json={
-                    "remove_health_debris": True,
-                    "remove_stale_sessions": False,
-                    "dry_run": True,
-                },
-            )
-
-    assert response.status_code == 200
-    data = response.json()
-    assert data["dry_run"] is True
-    # Candidates are returned
-    assert data["debris_memories_affected"] == 4
-    assert set(data["debris_user_ids"]) == {"_healthcheck_probe", "_test_e2e"}
-    assert data["stale_sessions_affected"] == 0
-    # Dry-run: DELETE helper must NOT have been called
-    mock_delete.assert_not_called()
-
-
-@pytest.mark.asyncio
-async def test_cleanup_live_execution_deletes_debris():
-    """
-    Live execution (dry_run=False) must call the delete helper and return the
-    count of deleted rows.
-    Covers: one live-execution test per spec requirement (TSK-4.11).
-    """
-    with (
-        patch(
-            "vds_memory_orchestrator.service.api.v1.cleanup._find_debris_memories",
-            new=AsyncMock(return_value=(["_healthcheck_probe"], 5)),
-        ),
-        patch(
-            "vds_memory_orchestrator.service.api.v1.cleanup._delete_debris_memories",
-            new=AsyncMock(return_value=5),
-        ) as mock_delete,
-    ):
-        async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
-            response = await client.post(
-                "/api/v1/cleanup",
-                json={
-                    "remove_health_debris": True,
-                    "remove_stale_sessions": False,
-                    "dry_run": False,
-                },
-            )
-
-    assert response.status_code == 200
-    data = response.json()
-    assert data["dry_run"] is False
-    assert data["debris_memories_affected"] == 5
-    assert data["debris_user_ids"] == ["_healthcheck_probe"]
-    # Live run: DELETE helper must have been called with the user IDs
-    mock_delete.assert_called_once_with(["_healthcheck_probe"])
-
-
-@pytest.mark.asyncio
-async def test_cleanup_remove_health_debris_reports_debris_sessions():
-    """
-    When remove_health_debris=True the response must include debris_sessions_affected.
-    This covers Finding 3 (AC-5.1): cleanup must also purge Redis sessions for
-    internal users, not only their Postgres memories.
-    Both dry-run and live paths are tested via the patched helper.
-    """
-    with (
-        patch(
-            "vds_memory_orchestrator.service.api.v1.cleanup._find_debris_memories",
-            new=AsyncMock(return_value=(["_healthcheck_probe", "_test_worker"], 7)),
-        ),
-        patch(
-            "vds_memory_orchestrator.service.api.v1.cleanup._delete_debris_memories",
-            new=AsyncMock(return_value=7),
-        ),
-        patch(
-            "vds_memory_orchestrator.service.api.v1.cleanup._delete_debris_sessions",
-            new=AsyncMock(return_value=3),
-        ) as mock_del_sessions,
-    ):
-        async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
-            # Live run so _delete_debris_sessions is called (not just found)
-            response = await client.post(
-                "/api/v1/cleanup",
-                json={
-                    "remove_health_debris": True,
-                    "remove_stale_sessions": False,
-                    "dry_run": False,
-                },
-            )
-
-    assert response.status_code == 200
-    data = response.json()
-    # debris_sessions_affected must be present and reflect the helper return value
-    assert "debris_sessions_affected" in data, (
-        "CleanupResponse must include debris_sessions_affected"
-    )
-    assert data["debris_sessions_affected"] == 3
-    # Helper must have been called with the user IDs found by _find_debris_memories
-    mock_del_sessions.assert_called_once_with(["_healthcheck_probe", "_test_worker"], dry_run=False)