@ngocsangairvds/vsaf 3.2.13 → 3.2.15

package/tools/vds-scripts/multi_agent_orchestrator/src/vds_multi_agent_orchestrator/conflict_resolver.py

@@ -1,462 +0,0 @@
-"""Conflict resolution and file locking for multi-agent coordination."""
-
-import asyncio
-import json
-import logging
-from datetime import UTC, datetime
-from typing import Any
-
-from .config import get_settings
-from .models import (
-    ConflictInfo,
-    ImportanceLevel,
-    MemoryCategory,
-    MemoryCreateRequest,
-)
-from .redis_client import get_redis_client
-from .vds_ai_memory_client import get_vds_ai_memory_client
-
-logger = logging.getLogger(__name__)
-
-
-class ConflictResolver:
-    """File-level conflict resolution with Redis-based locking."""
-
-    def __init__(self):
-        self.settings = get_settings()
-
-    async def acquire_file_lock(self, file_path: str, agent_id: str, operation: str) -> bool:
-        """Acquire exclusive lock on a file."""
-        redis_client = await get_redis_client()
-        vds_ai_memory = await get_vds_ai_memory_client()
-
-        lock_key = f"file_lock:{file_path}"
-        lock_data = {
-            "agent_id": agent_id,
-            "operation": operation,
-            "acquired_at": datetime.now(UTC).isoformat(),
-            "file_path": file_path,
-        }
-
-        # Try to acquire lock with timeout
-        lock_timeout = self.settings.file_lock_timeout_minutes * 60
-        success = await redis_client.set(
-            lock_key,
-            json.dumps(lock_data),
-            nx=True,  # Only set if key doesn't exist
-            ex=lock_timeout,  # Expire after timeout
-        )
-
-        if success:
-            # Store lock acquisition in VDS-AI-Memory
-            session_id = f"multi_agent_session_{datetime.now().strftime('%Y%m%d')}"
-            await vds_ai_memory.create_memory(
-                MemoryCreateRequest(
-                    content=f"Agent {agent_id} acquired file lock for {file_path} (operation: {operation})",
-                    category=MemoryCategory.BUSINESS_LOGIC,
-                    importance=ImportanceLevel.MEDIUM,
-                    session_id=session_id,
-                    user_id="multi_agent_coordinator",
-                    metadata={
-                        "org_id": "vds-internal",
-                        "agent_id": agent_id,
-                        "file_path": file_path,
-                        "operation": operation,
-                        "event_type": "file_lock_acquired",
-                    },
-                )
-            )
-
-            logger.debug(f"Agent {agent_id} acquired lock on {file_path}")
-            return True
-        else:
-            # Lock already exists, check who owns it
-            existing_lock = await redis_client.get(lock_key)
-            if existing_lock:
-                lock_info = json.loads(existing_lock)
-                logger.warning(
-                    f"Agent {agent_id} failed to acquire lock on {file_path} - owned by {lock_info.get('agent_id')}"
-                )
-
-            return False
-
-    async def release_file_lock(self, file_path: str, agent_id: str) -> bool:
-        """Release file lock."""
-        redis_client = await get_redis_client()
-        vds_ai_memory = await get_vds_ai_memory_client()
-
-        lock_key = f"file_lock:{file_path}"
-
-        # Verify agent owns the lock before releasing
-        lock_data = await redis_client.get(lock_key)
-        if lock_data:
-            lock_info = json.loads(lock_data)
-            if lock_info.get("agent_id") == agent_id:
-                await redis_client.delete(lock_key)
-
-                # Store lock release in VDS-AI-Memory
-                session_id = f"multi_agent_session_{datetime.now().strftime('%Y%m%d')}"
-                await vds_ai_memory.create_memory(
-                    MemoryCreateRequest(
-                        content=f"Agent {agent_id} released file lock for {file_path}",
-                        category=MemoryCategory.BUSINESS_LOGIC,
-                        importance=ImportanceLevel.LOW,
-                        session_id=session_id,
-                        user_id="multi_agent_coordinator",
-                        metadata={
-                            "org_id": "vds-internal",
-                            "agent_id": agent_id,
-                            "file_path": file_path,
-                            "event_type": "file_lock_released",
-                        },
-                    )
-                )
-
-                logger.debug(f"Agent {agent_id} released lock on {file_path}")
-                return True
-            else:
-                logger.warning(
-                    f"Agent {agent_id} attempted to release lock on {file_path} owned by {lock_info.get('agent_id')}"
-                )
-                return False
-
-        # Lock doesn't exist (maybe expired)
-        logger.debug(f"Lock on {file_path} not found (may have expired)")
-        return True
-
-    async def check_file_lock(self, file_path: str) -> dict[str, Any] | None:
-        """Check if file is locked and get lock information."""
-        redis_client = await get_redis_client()
-
-        lock_key = f"file_lock:{file_path}"
-        lock_data = await redis_client.get(lock_key)
-
-        if lock_data:
-            lock_info = json.loads(lock_data)
-            acquired_at = datetime.fromisoformat(lock_info["acquired_at"])
-            duration = (datetime.now(UTC) - acquired_at).total_seconds() / 60  # minutes
-
-            return {
-                "locked": True,
-                "agent_id": lock_info["agent_id"],
-                "operation": lock_info["operation"],
-                "acquired_at": lock_info["acquired_at"],
-                "duration_minutes": duration,
-                "ttl_seconds": await redis_client.ttl(lock_key),
-            }
-
-        return {"locked": False}
-
-    async def detect_conflicts(self) -> list[ConflictInfo]:
-        """Detect potential conflicts in concurrent operations."""
-        redis_client = await get_redis_client()
-
-        conflicts = []
-
-        # Check for long-held locks
-        lock_keys = await redis_client.keys("file_lock:*")
-
-        for lock_key in lock_keys:
-            lock_data = await redis_client.get(lock_key)
-            if not lock_data:
-                continue
-
-            lock_info = json.loads(lock_data)
-            acquired_at = datetime.fromisoformat(lock_info["acquired_at"])
-            duration_minutes = (datetime.now(UTC) - acquired_at).total_seconds() / 60
-
-            # Flag locks held for more than 20 minutes as potential conflicts
-            if duration_minutes > 20:
-                file_path = lock_key.replace("file_lock:", "")
-                conflicts.append(
-                    ConflictInfo(
-                        conflict_type="long_held_lock",
-                        file_path=file_path,
-                        agent_id=lock_info["agent_id"],
-                        operation=lock_info["operation"],
-                        duration_minutes=duration_minutes,
-                        details={
-                            "acquired_at": lock_info["acquired_at"],
-                            "ttl_seconds": await redis_client.ttl(lock_key),
-                        },
-                    )
-                )
-
-        # Check for agents with multiple locks (potential deadlock)
-        agent_locks = {}
-        for lock_key in lock_keys:
-            lock_data = await redis_client.get(lock_key)
-            if lock_data:
-                lock_info = json.loads(lock_data)
-                agent_id = lock_info["agent_id"]
-                if agent_id not in agent_locks:
-                    agent_locks[agent_id] = []
-                agent_locks[agent_id].append(lock_key.replace("file_lock:", ""))
-
-        # Flag agents with more than 3 simultaneous locks
-        for agent_id, locked_files in agent_locks.items():
-            if len(locked_files) > 3:
-                conflicts.append(
-                    ConflictInfo(
-                        conflict_type="multiple_locks",
-                        file_path="multiple",
-                        agent_id=agent_id,
-                        operation="multiple",
-                        duration_minutes=0,
-                        details={"locked_files": locked_files, "lock_count": len(locked_files)},
-                    )
-                )
-
-        return conflicts
-
-    async def resolve_conflict(self, conflict: ConflictInfo, resolution_strategy: str):
-        """Resolve detected conflict."""
-        redis_client = await get_redis_client()
-        vds_ai_memory = await get_vds_ai_memory_client()
-
-        session_id = f"multi_agent_session_{datetime.now().strftime('%Y%m%d')}"
-
-        if conflict.conflict_type == "long_held_lock":
-            if resolution_strategy == "force_release":
-                # Force release the lock
-                lock_key = f"file_lock:{conflict.file_path}"
-                await redis_client.delete(lock_key)
-
-                # Store conflict resolution in VDS-AI-Memory
-                await vds_ai_memory.create_memory(
-                    MemoryCreateRequest(
-                        content=f"Conflict resolved: Force released lock on {conflict.file_path} from agent {conflict.agent_id} after {conflict.duration_minutes:.1f} minutes",
-                        category=MemoryCategory.BUSINESS_LOGIC,
-                        importance=ImportanceLevel.CRITICAL,
-                        force_mem0=True,
-                        session_id=session_id,
-                        user_id="multi_agent_coordinator",
-                        metadata={
-                            "org_id": "vds-internal",
-                            "conflict_type": conflict.conflict_type,
-                            "resolution_strategy": resolution_strategy,
-                            "affected_agent": conflict.agent_id,
-                            "file_path": conflict.file_path,
-                            "duration_minutes": conflict.duration_minutes,
-                            "event_type": "conflict_resolution",
-                        },
-                    )
-                )
-
-                logger.warning(
-                    f"Force released lock on {conflict.file_path} from agent {conflict.agent_id}"
-                )
-
-            elif resolution_strategy == "extend_timeout":
-                # Extend lock timeout
-                lock_key = f"file_lock:{conflict.file_path}"
-                await redis_client.expire(lock_key, self.settings.file_lock_timeout_minutes * 60)
-
-                logger.info(
-                    f"Extended timeout for lock on {conflict.file_path} by agent {conflict.agent_id}"
-                )
-
-        elif conflict.conflict_type == "multiple_locks":
-            if resolution_strategy == "release_oldest":
-                # Release the oldest locks for this agent
-                locked_files = conflict.details.get("locked_files", [])
-
-                # Get lock timestamps and sort by age
-                lock_ages = []
-                for file_path in locked_files:
-                    lock_key = f"file_lock:{file_path}"
-                    lock_data = await redis_client.get(lock_key)
-                    if lock_data:
-                        lock_info = json.loads(lock_data)
-                        acquired_at = datetime.fromisoformat(lock_info["acquired_at"])
-                        lock_ages.append((file_path, acquired_at))
-
-                # Sort by age (oldest first) and release half of them
-                lock_ages.sort(key=lambda x: x[1])
-                locks_to_release = lock_ages[: len(lock_ages) // 2]
-
-                for file_path, _ in locks_to_release:
-                    await redis_client.delete(f"file_lock:{file_path}")
-
-                # Store resolution
-                await vds_ai_memory.create_memory(
-                    MemoryCreateRequest(
-                        content=f"Conflict resolved: Released {len(locks_to_release)} oldest locks from agent {conflict.agent_id}",
-                        category=MemoryCategory.BUSINESS_LOGIC,
-                        importance=ImportanceLevel.HIGH,
-                        session_id=session_id,
-                        user_id="multi_agent_coordinator",
-                        metadata={
-                            "org_id": "vds-internal",
-                            "conflict_type": conflict.conflict_type,
-                            "resolution_strategy": resolution_strategy,
-                            "affected_agent": conflict.agent_id,
-                            "released_locks": [fp for fp, _ in locks_to_release],
-                            "event_type": "conflict_resolution",
-                        },
-                    )
-                )
-
-                logger.warning(
-                    f"Released {len(locks_to_release)} oldest locks from agent {conflict.agent_id}"
-                )
-
-    async def get_lock_statistics(self) -> dict[str, Any]:
-        """Get file lock statistics."""
-        redis_client = await get_redis_client()
-
-        lock_keys = await redis_client.keys("file_lock:*")
-
-        if not lock_keys:
-            return {
-                "total_locks": 0,
-                "agents_with_locks": 0,
-                "average_lock_duration_minutes": 0,
-                "longest_lock_duration_minutes": 0,
-                "locks_by_agent": {},
-                "locks_by_operation": {},
-            }
-
-        agent_locks = {}
-        operation_locks = {}
-        durations = []
-
-        for lock_key in lock_keys:
-            lock_data = await redis_client.get(lock_key)
-            if not lock_data:
-                continue
-
-            lock_info = json.loads(lock_data)
-            agent_id = lock_info["agent_id"]
-            operation = lock_info["operation"]
-            acquired_at = datetime.fromisoformat(lock_info["acquired_at"])
-            duration_minutes = (datetime.now(UTC) - acquired_at).total_seconds() / 60
-
-            # Count by agent
-            if agent_id not in agent_locks:
-                agent_locks[agent_id] = 0
-            agent_locks[agent_id] += 1
-
-            # Count by operation
-            if operation not in operation_locks:
-                operation_locks[operation] = 0
-            operation_locks[operation] += 1
-
-            durations.append(duration_minutes)
-
-        return {
-            "total_locks": len(lock_keys),
-            "agents_with_locks": len(agent_locks),
-            "average_lock_duration_minutes": sum(durations) / len(durations) if durations else 0,
-            "longest_lock_duration_minutes": max(durations) if durations else 0,
-            "locks_by_agent": agent_locks,
-            "locks_by_operation": operation_locks,
-        }
-
-    async def cleanup_expired_locks(self):
-        """Clean up expired locks (Redis should handle this automatically, but this is a safety net)."""
-        redis_client = await get_redis_client()
-
-        lock_keys = await redis_client.keys("file_lock:*")
-        cleaned_count = 0
-
-        for lock_key in lock_keys:
-            ttl = await redis_client.ttl(lock_key)
-            if ttl == -1:  # Key exists but has no expiration
-                # This shouldn't happen, but clean it up
-                await redis_client.delete(lock_key)
-                cleaned_count += 1
-                logger.warning(f"Cleaned up lock without expiration: {lock_key}")
-
-        if cleaned_count > 0:
-            logger.info(f"Cleaned up {cleaned_count} locks without expiration")
-
-        return cleaned_count
-
-    async def start_conflict_monitor(self):
-        """Start background task to monitor and resolve conflicts."""
-
-        async def monitor_loop():
-            while True:
-                try:
-                    # Detect conflicts
-                    conflicts = await self.detect_conflicts()
-
-                    # Auto-resolve critical conflicts
-                    for conflict in conflicts:
-                        if (
-                            conflict.conflict_type == "long_held_lock"
-                            and conflict.duration_minutes > 30
-                        ):
-                            # Auto-resolve locks held for more than 30 minutes
-                            await self.resolve_conflict(conflict, "force_release")
-                        elif (
-                            conflict.conflict_type == "multiple_locks"
-                            and len(conflict.details.get("locked_files", [])) > 5
-                        ):
-                            # Auto-resolve agents with more than 5 locks
-                            await self.resolve_conflict(conflict, "release_oldest")
-
-                    # Clean up expired locks
-                    await self.cleanup_expired_locks()
-
-                    # Wait before next check
-                    await asyncio.sleep(self.settings.conflict_detection_interval_minutes * 60)
-
-                except Exception as e:
-                    logger.exception(f"Error in conflict monitor: {e}")
-                    await asyncio.sleep(60)  # Short delay before retry
-
-        # Start monitor task
-        asyncio.create_task(monitor_loop())
-        logger.info("Started conflict resolution monitor")
-
-    async def get_conflict_health(self) -> dict[str, Any]:
-        """Get conflict resolution system health."""
-        try:
-            conflicts = await self.detect_conflicts()
-            lock_stats = await self.get_lock_statistics()
-
-            # Calculate health score
-            health_score = 1.0
-
-            critical_conflicts = len(
-                [
-                    c
-                    for c in conflicts
-                    if c.conflict_type == "long_held_lock" and c.duration_minutes > 30
-                ]
-            )
-            if critical_conflicts > 0:
-                health_score = max(0.3, health_score - (critical_conflicts * 0.2))
-
-            multiple_lock_conflicts = len(
-                [c for c in conflicts if c.conflict_type == "multiple_locks"]
-            )
-            if multiple_lock_conflicts > 0:
-                health_score = max(0.5, health_score - (multiple_lock_conflicts * 0.1))
-
-            return {
-                "status": "healthy"
-                if health_score > 0.7
-                else "degraded"
-                if health_score > 0.3
-                else "unhealthy",
-                "health_score": health_score,
-                "conflicts_detected": len(conflicts),
-                "critical_conflicts": critical_conflicts,
-                "lock_statistics": lock_stats,
-                "conflicts": [
-                    {
-                        "type": c.conflict_type,
-                        "agent_id": c.agent_id,
-                        "file_path": c.file_path,
-                        "duration_minutes": c.duration_minutes,
-                        "details": c.details,
-                    }
-                    for c in conflicts
-                ],
-            }
-
-        except Exception as e:
-            return {"status": "unhealthy", "health_score": 0.0, "error": str(e)}