mdan-cli 2.5.1 → 2.5.2

package/app/packs/db-optimization/agents/indexing-specialist/agent.py

@@ -0,0 +1,713 @@
+"""
+Indexing Specialist Agent
+
+Specialized agent for database index design, optimization, and maintenance strategies.
+"""
+
+from typing import Dict, Any, List, Optional
+from dataclasses import dataclass
+import asyncio
+
+
+@dataclass
+class IndexingRequest:
+    """Request data structure for the Indexing Specialist agent."""
+
+    action: str
+    db_type: Optional[str] = None
+    table_name: Optional[str] = None
+    query_patterns: Optional[List[str]] = None
+    schema_info: Optional[Dict[str, Any]] = None
+    current_indexes: Optional[List[Dict[str, Any]]] = None
+    workload_type: Optional[str] = None
+
+
+class IndexingSpecialistAgent:
+    """
+    Indexing Specialist - Expert in database index design and optimization.
+
+    Specialized in designing optimal indexing strategies, analyzing index usage,
+    and providing index maintenance recommendations. Expert in various index types,
+    covering indexes, partial indexes, and index optimization techniques.
+    """
+
+    def __init__(self):
+        """Initialize the Indexing Specialist agent."""
+        self.name = "Indexing Specialist"
+        self.title = "Database Index Design and Optimization Expert"
+        self.icon = "📑"
+        self.capabilities = [
+            "Index design and strategy",
+            "Index type selection (B-tree, Hash, GIN, GiST, etc.)",
+            "Covering index design",
+            "Partial index optimization",
+            "Composite index ordering",
+            "Index usage analysis",
+            "Index maintenance strategies",
+            "Index fragmentation management",
+            "Index size optimization",
+            "Index performance monitoring",
+        ]
+        self.role = "Indexing Expert"
+        self.identity = (
+            "Expert indexing specialist with deep knowledge of database index "
+            "structures, query optimization, and index maintenance strategies. "
+            "Specialized in designing efficient indexing solutions for various "
+            "workloads and database systems."
+        )
+        self.communication_style = (
+            "Strategic and analytical, focusing on index design principles, "
+            "query patterns, and performance trade-offs. Provides specific "
+            "index definitions with explanations of their purpose and impact."
+        )
+        self.principles = [
+            "Design indexes based on actual query patterns",
+            "Prioritize indexes for the most frequent and expensive queries",
+            "Consider the write vs. read trade-off",
+            "Use appropriate index types for the data and queries",
+            "Avoid over-indexing to minimize write overhead",
+            "Monitor index usage and remove unused indexes",
+            "Regularly maintain indexes to prevent fragmentation",
+            "Consider index size and memory requirements",
+        ]
+
+    async def process(self, request: IndexingRequest) -> Dict[str, Any]:
+        """
+        Process the indexing request and return a response.
+
+        Args:
+            request: The indexing request to process
+
+        Returns:
+            A dictionary containing the indexing analysis and recommendations
+        """
+        action = request.action.lower()
+
+        if action == "design":
+            return await self._design_indexes(request)
+        elif action == "analyze":
+            return await self._analyze_indexes(request)
+        elif action == "optimize":
+            return await self._optimize_indexes(request)
+        elif action == "maintain":
+            return await self._maintain_indexes(request)
+        elif action == "recommend":
+            return await self._recommend_strategy(request)
+        else:
+            return await self._handle_unknown_action(request)
+
+    async def _design_indexes(self, request: IndexingRequest) -> Dict[str, Any]:
+        """
+        Design indexes for tables.
+
+        Args:
+            request: The request to process
+
+        Returns:
+            Index design recommendations
+        """
+        db_type = request.db_type or "PostgreSQL"
+        table_name = request.table_name or "users"
+        query_patterns = request.query_patterns or []
+        schema_info = request.schema_info or {}
+
+        design = {
+            "status": "success",
+            "action": "design",
+            "db_type": db_type,
+            "table_name": table_name,
+            "recommended_indexes": self._generate_index_recommendations(
+                table_name, query_patterns, schema_info, db_type
+            ),
+            "index_types": self._recommend_index_types(query_patterns, db_type),
+            "index_ordering": self._recommend_column_ordering(query_patterns, db_type),
+            "covering_indexes": self._design_covering_indexes(
+                query_patterns, schema_info, db_type
+            ),
+            "partial_indexes": self._design_partial_indexes(query_patterns, db_type),
+            "expected_impact": self._estimate_index_impact(query_patterns, db_type),
+        }
+
+        return design
+
+    async def _analyze_indexes(self, request: IndexingRequest) -> Dict[str, Any]:
+        """
+        Analyze existing indexes.
+
+        Args:
+            request: The request to process
+
+        Returns:
+            Index analysis results
+        """
+        db_type = request.db_type or "PostgreSQL"
+        current_indexes = request.current_indexes or []
+        query_patterns = request.query_patterns or []
+
+        analysis = {
+            "status": "success",
+            "action": "analyze",
+            "db_type": db_type,
+            "current_indexes": current_indexes,
+            "index_usage": self._analyze_index_usage(current_indexes, query_patterns),
+            "unused_indexes": self._identify_unused_indexes(
+                current_indexes, query_patterns
+            ),
+            "duplicate_indexes": self._identify_duplicate_indexes(current_indexes),
+            "inefficient_indexes": self._identify_inefficient_indexes(
+                current_indexes, query_patterns
+            ),
+            "missing_indexes": self._identify_missing_indexes(
+                current_indexes, query_patterns
+            ),
+            "recommendations": self._generate_analysis_recommendations(
+                current_indexes, query_patterns
+            ),
+        }
+
+        return analysis
+
+    async def _optimize_indexes(self, request: IndexingRequest) -> Dict[str, Any]:
+        """
+        Optimize existing indexes.
+
+        Args:
+            request: The request to process
+
+        Returns:
+            Index optimization recommendations
+        """
+        db_type = request.db_type or "PostgreSQL"
+        current_indexes = request.current_indexes or []
+        query_patterns = request.query_patterns or []
+
+        optimization = {
+            "status": "success",
+            "action": "optimize",
+            "db_type": db_type,
+            "indexes_to_remove": self._recommend_index_removal(
+                current_indexes, query_patterns
+            ),
+            "indexes_to_modify": self._recommend_index_modifications(
+                current_indexes, query_patterns
+            ),
+            "indexes_to_add": self._recommend_new_indexes(
+                current_indexes, query_patterns
+            ),
+            "rebuild_strategy": self._recommend_rebuild_strategy(db_type),
+            "expected_improvement": self._estimate_optimization_improvement(db_type),
+        }
+
+        return optimization
+
+    async def _maintain_indexes(self, request: IndexingRequest) -> Dict[str, Any]:
+        """
+        Provide index maintenance recommendations.
+
+        Args:
+            request: The request to process
+
+        Returns:
+            Index maintenance recommendations
+        """
+        db_type = request.db_type or "PostgreSQL"
+
+        maintenance = {
+            "status": "success",
+            "action": "maintain",
+            "db_type": db_type,
+            "maintenance_tasks": self._get_maintenance_tasks(db_type),
+            "rebuild_schedule": self._recommend_rebuild_schedule(db_type),
+            "fragmentation_management": self._manage_fragmentation(db_type),
+            "statistics_update": self._update_statistics(db_type),
+            "monitoring": self._setup_index_monitoring(db_type),
+        }
+
+        return maintenance
+
+    async def _recommend_strategy(self, request: IndexingRequest) -> Dict[str, Any]:
+        """
+        Recommend overall indexing strategy.
+
+        Args:
+            request: The request to process
+
+        Returns:
+            Indexing strategy recommendations
+        """
+        db_type = request.db_type or "PostgreSQL"
+        workload_type = request.workload_type or "mixed"
+        query_patterns = request.query_patterns or []
+
+        strategy = {
+            "status": "success",
+            "action": "recommend",
+            "db_type": db_type,
+            "workload_type": workload_type,
+            "overall_strategy": self._develop_indexing_strategy(workload_type, db_type),
+            "indexing_principles": self._define_indexing_principles(workload_type),
+            "implementation_phases": self._plan_implementation(workload_type, db_type),
+            "monitoring_plan": self._create_monitoring_plan(db_type),
+            "success_metrics": self._define_success_metrics(workload_type),
+        }
+
+        return strategy
+
+    async def _handle_unknown_action(self, request: IndexingRequest) -> Dict[str, Any]:
+        """
+        Handle unknown action.
+
+        Args:
+            request: The request to process
+
+        Returns:
+            Error response
+        """
+        return {
+            "status": "error",
+            "action": request.action,
+            "message": f"Unknown action: {request.action}",
+            "available_actions": [
+                "design",
+                "analyze",
+                "optimize",
+                "maintain",
+                "recommend",
+            ],
+        }
+
+    def _generate_index_recommendations(
+        self,
+        table_name: str,
+        query_patterns: List[str],
+        schema_info: Dict[str, Any],
+        db_type: str,
+    ) -> List[Dict[str, Any]]:
+        """Generate index recommendations."""
+        recommendations = []
+
+        # Primary key index
+        recommendations.append(
+            {
+                "name": f"pk_{table_name}",
+                "table": table_name,
+                "columns": ["id"],
+                "type": "btree",
+                "unique": True,
+                "primary": True,
+                "reason": "Primary key for unique identification",
+            }
+        )
+
+        # Common query pattern indexes
+        if any("WHERE email" in q for q in query_patterns):
+            recommendations.append(
+                {
+                    "name": f"idx_{table_name}_email",
+                    "table": table_name,
+                    "columns": ["email"],
+                    "type": "btree",
+                    "unique": True,
+                    "reason": "Frequent lookups by email",
+                }
+            )
+
+        if any("WHERE user_id" in q for q in query_patterns):
+            recommendations.append(
+                {
+                    "name": f"idx_{table_name}_user_id",
+                    "table": table_name,
+                    "columns": ["user_id"],
+                    "type": "btree",
+                    "reason": "Foreign key for user relationships",
+                }
+            )
+
+        if any("ORDER BY created_at" in q for q in query_patterns):
+            recommendations.append(
+                {
+                    "name": f"idx_{table_name}_created_at",
+                    "table": table_name,
+                    "columns": ["created_at"],
+                    "type": "btree",
+                    "reason": "Frequent sorting by creation date",
+                }
+            )
+
+        return recommendations
+
+    def _recommend_index_types(
+        self, query_patterns: List[str], db_type: str
+    ) -> List[Dict[str, Any]]:
+        """Recommend index types."""
+        return [
+            {
+                "type": "btree",
+                "use_case": "Equality and range queries, ORDER BY",
+                "when_to_use": "Most common index type, good for general purpose",
+            },
+            {
+                "type": "hash",
+                "use_case": "Equality queries only",
+                "when_to_use": "When only equality comparisons are needed",
+            },
+            {
+                "type": "gin",
+                "use_case": "JSONB, arrays, full-text search",
+                "when_to_use": "For composite data types and containment queries",
+            },
+            {
+                "type": "gist",
+                "use_case": "Geometric data, ranges",
+                "when_to_use": "For spatial and range queries",
+            },
+        ]
+
+    def _recommend_column_ordering(
+        self, query_patterns: List[str], db_type: str
+    ) -> List[str]:
+        """Recommend column ordering in composite indexes."""
+        return [
+            "Most selective column first",
+            "Equality columns before range columns",
+            "Columns used in WHERE before ORDER BY",
+            "Consider query frequency and selectivity",
+        ]
+
+    def _design_covering_indexes(
+        self, query_patterns: List[str], schema_info: Dict[str, Any], db_type: str
+    ) -> List[Dict[str, Any]]:
+        """Design covering indexes."""
+        return [
+            {
+                "name": "idx_covering_user_orders",
+                "table": "orders",
+                "columns": ["user_id", "status", "created_at", "total"],
+                "type": "btree",
+                "reason": "Covering index for user order queries - enables index-only scan",
+                "benefit": "Eliminates table access, improves query performance",
+            },
+        ]
+
+    def _design_partial_indexes(
+        self, query_patterns: List[str], db_type: str
+    ) -> List[Dict[str, Any]]:
+        """Design partial indexes."""
+        return [
+            {
+                "name": "idx_active_users",
+                "table": "users",
+                "columns": ["email"],
+                "type": "btree",
+                "where": "active = true",
+                "reason": "Index only active users, smaller and faster",
+                "benefit": "Reduced index size, faster queries on active users",
+            },
+        ]
+
+    def _estimate_index_impact(
+        self, query_patterns: List[str], db_type: str
+    ) -> Dict[str, str]:
+        """Estimate index impact."""
+        return {
+            "query_performance": "50-90% improvement for indexed queries",
+            "write_performance": "10-30% slowdown for INSERT/UPDATE/DELETE",
+            "storage_overhead": "20-50% of table size",
+            "memory_usage": "Indexes cached in memory for faster access",
+        }
+
+    def _analyze_index_usage(
+        self, current_indexes: List[Dict[str, Any]], query_patterns: List[str]
+    ) -> Dict[str, Any]:
+        """Analyze index usage."""
+        return {
+            "total_indexes": len(current_indexes),
+            "frequently_used": len(current_indexes) // 2,
+            "rarely_used": len(current_indexes) // 4,
+            "unused": len(current_indexes) // 4,
+        }
+
+    def _identify_unused_indexes(
+        self, current_indexes: List[Dict[str, Any]], query_patterns: List[str]
+    ) -> List[Dict[str, Any]]:
+        """Identify unused indexes."""
+        return [
+            {
+                "name": "idx_unused_column",
+                "table": "users",
+                "reason": "No queries use this index",
+                "recommendation": "Consider dropping to reduce write overhead",
+            },
+        ]
+
+    def _identify_duplicate_indexes(
+        self, current_indexes: List[Dict[str, Any]]
+    ) -> List[Dict[str, Any]]:
+        """Identify duplicate indexes."""
+        return [
+            {
+                "duplicate_1": "idx_users_email",
+                "duplicate_2": "idx_users_email_unique",
+                "reason": "Both indexes on same column",
+                "recommendation": "Keep only the unique index",
+            },
+        ]
+
+    def _identify_inefficient_indexes(
+        self, current_indexes: List[Dict[str, Any]], query_patterns: List[str]
+    ) -> List[Dict[str, Any]]:
+        """Identify inefficient indexes."""
+        return [
+            {
+                "name": "idx_inefficient",
+                "table": "users",
+                "issue": "Low selectivity column",
+                "reason": "Column has few distinct values",
+                "recommendation": "Consider removing or using partial index",
+            },
+        ]
+
+    def _identify_missing_indexes(
+        self, current_indexes: List[Dict[str, Any]], query_patterns: List[str]
+    ) -> List[Dict[str, Any]]:
+        """Identify missing indexes."""
+        return [
+            {
+                "table": "orders",
+                "columns": ["user_id", "created_at"],
+                "reason": "Frequent query pattern not covered",
+                "priority": "high",
+            },
+        ]
+
+    def _generate_analysis_recommendations(
+        self, current_indexes: List[Dict[str, Any]], query_patterns: List[str]
+    ) -> List[Dict[str, Any]]:
+        """Generate analysis recommendations."""
+        return [
+            {
+                "action": "drop_unused",
+                "indexes": ["idx_unused_column"],
+                "benefit": "Reduce write overhead and storage",
+            },
+            {
+                "action": "add_missing",
+                "indexes": ["idx_orders_user_created"],
+                "benefit": "Improve query performance",
+            },
+        ]
+
+    def _recommend_index_removal(
+        self, current_indexes: List[Dict[str, Any]], query_patterns: List[str]
+    ) -> List[Dict[str, Any]]:
+        """Recommend index removal."""
+        return [
+            {
+                "name": "idx_unused_column",
+                "reason": "Unused index",
+                "impact": "Reduces write overhead",
+            },
+        ]
+
+    def _recommend_index_modifications(
+        self, current_indexes: List[Dict[str, Any]], query_patterns: List[str]
+    ) -> List[Dict[str, Any]]:
+        """Recommend index modifications."""
+        return [
+            {
+                "name": "idx_users_email",
+                "modification": "Add INCLUDE clause for covering",
+                "reason": "Enable index-only scan",
+            },
+        ]
+
+    def _recommend_new_indexes(
+        self, current_indexes: List[Dict[str, Any]], query_patterns: List[str]
+    ) -> List[Dict[str, Any]]:
+        """Recommend new indexes."""
+        return [
+            {
+                "name": "idx_orders_user_created",
+                "table": "orders",
+                "columns": ["user_id", "created_at"],
+                "type": "btree",
+                "reason": "Support frequent query pattern",
+            },
+        ]
+
+    def _recommend_rebuild_strategy(self, db_type: str) -> Dict[str, Any]:
+        """Recommend rebuild strategy."""
+        return {
+            "frequency": "monthly",
+            "method": "REINDEX CONCURRENTLY",
+            "maintenance_window": "low-traffic period",
+        }
+
+    def _estimate_optimization_improvement(self, db_type: str) -> Dict[str, str]:
+        """Estimate optimization improvement."""
+        return {
+            "query_performance": "20-40% improvement",
+            "storage": "10-20% reduction",
+            "write_performance": "10-15% improvement",
+        }
+
+    def _get_maintenance_tasks(self, db_type: str) -> List[Dict[str, Any]]:
+        """Get maintenance tasks."""
+        return [
+            {
+                "task": "REINDEX",
+                "frequency": "monthly",
+                "purpose": "Rebuild fragmented indexes",
+            },
+            {
+                "task": "ANALYZE",
+                "frequency": "weekly",
+                "purpose": "Update statistics for query planner",
+            },
+            {
+                "task": "VACUUM",
+                "frequency": "daily",
+                "purpose": "Reclaim space and update statistics",
+            },
+        ]
+
+    def _recommend_rebuild_schedule(self, db_type: str) -> Dict[str, Any]:
+        """Recommend rebuild schedule."""
+        return {
+            "full_rebuild": "monthly",
+            "partial_rebuild": "weekly",
+            "maintenance_window": "2:00 AM - 4:00 AM",
+        }
+
+    def _manage_fragmentation(self, db_type: str) -> List[str]:
+        """Manage fragmentation."""
+        return [
+            "Monitor index fragmentation levels",
+            "Rebuild indexes when fragmentation > 30%",
+            "Use FILLFACTOR to reduce fragmentation",
+        ]
+
+    def _update_statistics(self, db_type: str) -> List[str]:
+        """Update statistics."""
+        return [
+            "Run ANALYZE after significant data changes",
+            "Schedule automatic statistics updates",
+            "Monitor statistics accuracy",
+        ]
+
+    def _setup_index_monitoring(self, db_type: str) -> Dict[str, Any]:
+        """Set up index monitoring."""
+        return {
+            "metrics": [
+                "index usage count",
+                "index size",
+                "index scan time",
+                "index hit ratio",
+            ],
+            "alerts": [
+                "Unused index detected",
+                "Index fragmentation high",
+                "Index size growing rapidly",
+            ],
+        }
+
+    def _develop_indexing_strategy(
+        self, workload_type: str, db_type: str
+    ) -> Dict[str, Any]:
+        """Develop indexing strategy."""
+        strategies = {
+            "read_heavy": {
+                "approach": "Aggressive indexing",
+                "focus": "Optimize for query performance",
+                "trade_off": "Accept higher write overhead",
+            },
+            "write_heavy": {
+                "approach": "Conservative indexing",
+                "focus": "Minimize write impact",
+                "trade_off": "Accept slower queries",
+            },
+            "mixed": {
+                "approach": "Balanced indexing",
+                "focus": "Optimize for most frequent queries",
+                "trade_off": "Balance read and write performance",
+            },
+        }
+
+        return strategies.get(workload_type, strategies["mixed"])
+
+    def _define_indexing_principles(self, workload_type: str) -> List[str]:
+        """Define indexing principles."""
+        return [
+            "Index based on actual query patterns",
+            "Prioritize high-impact indexes",
+            "Monitor and adjust regularly",
+            "Consider the full lifecycle of indexes",
+        ]
+
+    def _plan_implementation(
+        self, workload_type: str, db_type: str
+    ) -> List[Dict[str, Any]]:
+        """Plan implementation phases."""
+        return [
+            {
+                "phase": 1,
+                "description": "Add high-priority indexes",
+                "duration": "1 week",
+            },
+            {
+                "phase": 2,
+                "description": "Monitor and adjust",
+                "duration": "2 weeks",
+            },
+            {
+                "phase": 3,
+                "description": "Remove unused indexes",
+                "duration": "1 week",
+            },
+        ]
+
+    def _create_monitoring_plan(self, db_type: str) -> Dict[str, Any]:
+        """Create monitoring plan."""
+        return {
+            "daily_checks": ["index usage", "index size"],
+            "weekly_reviews": ["index effectiveness", "query performance"],
+            "monthly_audits": ["index strategy review", "index cleanup"],
+        }
+
+    def _define_success_metrics(self, workload_type: str) -> List[Dict[str, Any]]:
+        """Define success metrics."""
+        return [
+            {
+                "metric": "query_performance",
+                "target": "50% improvement",
+                "measurement": "Average query time",
+            },
+            {
+                "metric": "index_usage",
+                "target": "> 80% of indexes used",
+                "measurement": "Index scan count",
+            },
+            {
+                "metric": "write_overhead",
+                "target": "< 20% slowdown",
+                "measurement": "Insert/update/delete time",
+            },
+        ]
+
+    def get_info(self) -> Dict[str, Any]:
+        """
+        Get agent information.
+
+        Returns:
+            Agent information dictionary
+        """
+        return {
+            "name": self.name,
+            "title": self.title,
+            "icon": self.icon,
+            "role": self.role,
+            "capabilities": self.capabilities,
+            "identity": self.identity,
+            "communication_style": self.communication_style,
+            "principles": self.principles,
+        }