aigroup-workflow 1.0.0

package/skills/jarvis/engineering-team/engineering-team/senior-data-engineer/scripts/etl_performance_optimizer.py

@@ -0,0 +1,1685 @@
+#!/usr/bin/env python3
+"""
+ETL Performance Optimizer
+Comprehensive ETL/ELT performance analysis and optimization tool.
+
+Features:
+- SQL query analysis and optimization recommendations
+- Spark job configuration analysis
+- Data skew detection and mitigation
+- Partition strategy recommendations
+- Join optimization suggestions
+- Memory and shuffle analysis
+- Cost estimation for cloud warehouses
+
+Usage:
+    python etl_performance_optimizer.py analyze-sql query.sql
+    python etl_performance_optimizer.py analyze-spark spark-history.json
+    python etl_performance_optimizer.py optimize-partition data_stats.json
+    python etl_performance_optimizer.py estimate-cost query.sql --warehouse snowflake
+"""
+
+import os
+import sys
+import json
+import re
+import argparse
+import logging
+import math
+from pathlib import Path
+from typing import Dict, List, Optional, Any, Tuple, Set
+from dataclasses import dataclass, field, asdict
+from datetime import datetime
+from collections import defaultdict
+from abc import ABC, abstractmethod
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+
+# =============================================================================
+# Data Classes
+# =============================================================================
+
+@dataclass
+class SQLQueryInfo:
+    """Parsed information about a SQL query"""
+    query_type: str  # SELECT, INSERT, UPDATE, DELETE, MERGE, CREATE
+    tables: List[str]
+    columns: List[str]
+    joins: List[Dict[str, str]]
+    where_conditions: List[str]
+    group_by: List[str]
+    order_by: List[str]
+    aggregations: List[str]
+    subqueries: int
+    distinct: bool
+    limit: Optional[int]
+    ctes: List[str]
+    window_functions: List[str]
+    estimated_complexity: str  # low, medium, high, very_high
+
+
+@dataclass
+class OptimizationRecommendation:
+    """A single optimization recommendation"""
+    category: str  # index, partition, join, filter, aggregation, memory, shuffle
+    severity: str  # critical, high, medium, low
+    title: str
+    description: str
+    current_issue: str
+    recommendation: str
+    expected_improvement: str
+    implementation: str
+    priority: int = 1
+
+
+@dataclass
+class SparkJobMetrics:
+    """Metrics from a Spark job"""
+    job_id: str
+    duration_ms: int
+    stages: int
+    tasks: int
+    shuffle_read_bytes: int
+    shuffle_write_bytes: int
+    input_bytes: int
+    output_bytes: int
+    peak_memory_bytes: int
+    gc_time_ms: int
+    failed_tasks: int
+    speculative_tasks: int
+    skew_ratio: float  # max_task_time / median_task_time
+
+
+@dataclass
+class PartitionStrategy:
+    """Recommended partition strategy"""
+    column: str
+    partition_type: str  # range, hash, list
+    num_partitions: Optional[int]
+    partition_size_mb: float
+    reasoning: str
+    implementation: str
+
+
+@dataclass
+class CostEstimate:
+    """Cost estimate for a query"""
+    warehouse: str
+    compute_cost: float
+    storage_cost: float
+    data_transfer_cost: float
+    total_cost: float
+    currency: str = "USD"
+    assumptions: List[str] = field(default_factory=list)
+
+
+# =============================================================================
+# SQL Parser
+# =============================================================================
+
+class SQLParser:
+    """Parse and analyze SQL queries"""
+
+    # Common SQL patterns
+    PATTERNS = {
+        'select': re.compile(r'\bSELECT\b', re.IGNORECASE),
+        'from': re.compile(r'\bFROM\b', re.IGNORECASE),
+        'join': re.compile(r'\b(INNER|LEFT|RIGHT|FULL|CROSS)?\s*JOIN\b', re.IGNORECASE),
+        'where': re.compile(r'\bWHERE\b', re.IGNORECASE),
+        'group_by': re.compile(r'\bGROUP\s+BY\b', re.IGNORECASE),
+        'order_by': re.compile(r'\bORDER\s+BY\b', re.IGNORECASE),
+        'having': re.compile(r'\bHAVING\b', re.IGNORECASE),
+        'distinct': re.compile(r'\bDISTINCT\b', re.IGNORECASE),
+        'limit': re.compile(r'\bLIMIT\s+(\d+)', re.IGNORECASE),
+        'cte': re.compile(r'\bWITH\b', re.IGNORECASE),
+        'subquery': re.compile(r'\(\s*SELECT\b', re.IGNORECASE),
+        'window': re.compile(r'\bOVER\s*\(', re.IGNORECASE),
+        'aggregation': re.compile(r'\b(COUNT|SUM|AVG|MIN|MAX|STDDEV|VARIANCE)\s*\(', re.IGNORECASE),
+        'insert': re.compile(r'\bINSERT\s+INTO\b', re.IGNORECASE),
+        'update': re.compile(r'\bUPDATE\b', re.IGNORECASE),
+        'delete': re.compile(r'\bDELETE\s+FROM\b', re.IGNORECASE),
+        'merge': re.compile(r'\bMERGE\s+INTO\b', re.IGNORECASE),
+        'create': re.compile(r'\bCREATE\s+(TABLE|VIEW|INDEX)\b', re.IGNORECASE),
+    }
+
+    def parse(self, sql: str) -> SQLQueryInfo:
+        """Parse a SQL query and extract information"""
+        # Clean up the query
+        sql = self._clean_sql(sql)
+
+        # Determine query type
+        query_type = self._detect_query_type(sql)
+
+        # Extract tables
+        tables = self._extract_tables(sql)
+
+        # Extract columns (for SELECT queries)
+        columns = self._extract_columns(sql) if query_type == 'SELECT' else []
+
+        # Extract joins
+        joins = self._extract_joins(sql)
+
+        # Extract WHERE conditions
+        where_conditions = self._extract_where_conditions(sql)
+
+        # Extract GROUP BY
+        group_by = self._extract_group_by(sql)
+
+        # Extract ORDER BY
+        order_by = self._extract_order_by(sql)
+
+        # Extract aggregations
+        aggregations = self._extract_aggregations(sql)
+
+        # Count subqueries
+        subqueries = len(self.PATTERNS['subquery'].findall(sql))
+
+        # Check for DISTINCT
+        distinct = bool(self.PATTERNS['distinct'].search(sql))
+
+        # Extract LIMIT
+        limit_match = self.PATTERNS['limit'].search(sql)
+        limit = int(limit_match.group(1)) if limit_match else None
+
+        # Extract CTEs
+        ctes = self._extract_ctes(sql)
+
+        # Extract window functions
+        window_functions = self._extract_window_functions(sql)
+
+        # Estimate complexity
+        complexity = self._estimate_complexity(
+            tables, joins, subqueries, aggregations, window_functions
+        )
+
+        return SQLQueryInfo(
+            query_type=query_type,
+            tables=tables,
+            columns=columns,
+            joins=joins,
+            where_conditions=where_conditions,
+            group_by=group_by,
+            order_by=order_by,
+            aggregations=aggregations,
+            subqueries=subqueries,
+            distinct=distinct,
+            limit=limit,
+            ctes=ctes,
+            window_functions=window_functions,
+            estimated_complexity=complexity
+        )
+
+    def _clean_sql(self, sql: str) -> str:
+        """Clean and normalize SQL"""
+        # Remove comments
+        sql = re.sub(r'--.*$', '', sql, flags=re.MULTILINE)
+        sql = re.sub(r'/\*.*?\*/', '', sql, flags=re.DOTALL)
+        # Normalize whitespace
+        sql = ' '.join(sql.split())
+        return sql
+
+    def _detect_query_type(self, sql: str) -> str:
+        """Detect the type of SQL query"""
+        sql_upper = sql.upper().strip()
+
+        if sql_upper.startswith('WITH') or sql_upper.startswith('SELECT'):
+            return 'SELECT'
+        elif self.PATTERNS['insert'].search(sql):
+            return 'INSERT'
+        elif self.PATTERNS['update'].search(sql):
+            return 'UPDATE'
+        elif self.PATTERNS['delete'].search(sql):
+            return 'DELETE'
+        elif self.PATTERNS['merge'].search(sql):
+            return 'MERGE'
+        elif self.PATTERNS['create'].search(sql):
+            return 'CREATE'
+        else:
+            return 'UNKNOWN'
+
+    def _extract_tables(self, sql: str) -> List[str]:
+        """Extract table names from SQL"""
+        tables = []
+
+        # FROM clause tables
+        from_pattern = re.compile(
+            r'\bFROM\s+([a-zA-Z_][a-zA-Z0-9_]*(?:\.[a-zA-Z_][a-zA-Z0-9_]*)?)',
+            re.IGNORECASE
+        )
+        tables.extend(from_pattern.findall(sql))
+
+        # JOIN clause tables
+        join_pattern = re.compile(
+            r'\bJOIN\s+([a-zA-Z_][a-zA-Z0-9_]*(?:\.[a-zA-Z_][a-zA-Z0-9_]*)?)',
+            re.IGNORECASE
+        )
+        tables.extend(join_pattern.findall(sql))
+
+        # INSERT INTO table
+        insert_pattern = re.compile(
+            r'\bINSERT\s+INTO\s+([a-zA-Z_][a-zA-Z0-9_]*(?:\.[a-zA-Z_][a-zA-Z0-9_]*)?)',
+            re.IGNORECASE
+        )
+        tables.extend(insert_pattern.findall(sql))
+
+        # UPDATE table
+        update_pattern = re.compile(
+            r'\bUPDATE\s+([a-zA-Z_][a-zA-Z0-9_]*(?:\.[a-zA-Z_][a-zA-Z0-9_]*)?)',
+            re.IGNORECASE
+        )
+        tables.extend(update_pattern.findall(sql))
+
+        return list(set(tables))
+
+    def _extract_columns(self, sql: str) -> List[str]:
+        """Extract column references from SELECT clause"""
+        # Find SELECT ... FROM
+        match = re.search(r'\bSELECT\s+(.*?)\s+FROM\b', sql, re.IGNORECASE | re.DOTALL)
+        if not match:
+            return []
+
+        select_clause = match.group(1)
+
+        # Handle SELECT *
+        if '*' in select_clause and 'COUNT(*)' not in select_clause.upper():
+            return ['*']
+
+        # Extract column names (simplified)
+        columns = []
+        for part in select_clause.split(','):
+            part = part.strip()
+            # Handle aliases
+            alias_match = re.search(r'\bAS\s+(\w+)\s*$', part, re.IGNORECASE)
+            if alias_match:
+                columns.append(alias_match.group(1))
+            else:
+                # Get the last identifier
+                col_match = re.search(r'([a-zA-Z_][a-zA-Z0-9_]*)(?:\s*$|\s+AS\b)', part, re.IGNORECASE)
+                if col_match:
+                    columns.append(col_match.group(1))
+
+        return columns
+
+    def _extract_joins(self, sql: str) -> List[Dict[str, str]]:
+        """Extract join information"""
+        joins = []
+
+        join_pattern = re.compile(
+            r'\b(INNER|LEFT\s+OUTER?|RIGHT\s+OUTER?|FULL\s+OUTER?|CROSS)?\s*JOIN\s+'
+            r'([a-zA-Z_][a-zA-Z0-9_.]*)\s*(?:AS\s+)?(\w+)?\s*'
+            r'(?:ON\s+(.+?))?(?=\s+(?:INNER|LEFT|RIGHT|FULL|CROSS|WHERE|GROUP|ORDER|HAVING|LIMIT|$))',
+            re.IGNORECASE | re.DOTALL
+        )
+
+        for match in join_pattern.finditer(sql):
+            join_type = match.group(1) or 'INNER'
+            table = match.group(2)
+            alias = match.group(3)
+            condition = match.group(4)
+
+            joins.append({
+                'type': join_type.strip().upper(),
+                'table': table,
+                'alias': alias,
+                'condition': condition.strip() if condition else None
+            })
+
+        return joins
+
+    def _extract_where_conditions(self, sql: str) -> List[str]:
+        """Extract WHERE clause conditions"""
+        # Find WHERE ... (GROUP BY | ORDER BY | HAVING | LIMIT | end)
+        match = re.search(
+            r'\bWHERE\s+(.*?)(?=\s+(?:GROUP\s+BY|ORDER\s+BY|HAVING|LIMIT)|$)',
+            sql, re.IGNORECASE | re.DOTALL
+        )
+        if not match:
+            return []
+
+        where_clause = match.group(1).strip()
+
+        # Split by AND/OR (simplified)
+        conditions = re.split(r'\s+AND\s+|\s+OR\s+', where_clause, flags=re.IGNORECASE)
+        return [c.strip() for c in conditions if c.strip()]
+
+    def _extract_group_by(self, sql: str) -> List[str]:
+        """Extract GROUP BY columns"""
+        match = re.search(
+            r'\bGROUP\s+BY\s+(.*?)(?=\s+(?:HAVING|ORDER\s+BY|LIMIT)|$)',
+            sql, re.IGNORECASE | re.DOTALL
+        )
+        if not match:
+            return []
+
+        group_clause = match.group(1).strip()
+        columns = [c.strip() for c in group_clause.split(',')]
+        return columns
+
+    def _extract_order_by(self, sql: str) -> List[str]:
+        """Extract ORDER BY columns"""
+        match = re.search(
+            r'\bORDER\s+BY\s+(.*?)(?=\s+LIMIT|$)',
+            sql, re.IGNORECASE | re.DOTALL
+        )
+        if not match:
+            return []
+
+        order_clause = match.group(1).strip()
+        columns = [c.strip() for c in order_clause.split(',')]
+        return columns
+
+    def _extract_aggregations(self, sql: str) -> List[str]:
+        """Extract aggregation functions used"""
+        agg_pattern = re.compile(
+            r'\b(COUNT|SUM|AVG|MIN|MAX|STDDEV|VARIANCE|MEDIAN|PERCENTILE_CONT|PERCENTILE_DISC)\s*\(',
+            re.IGNORECASE
+        )
+        return list(set(m.upper() for m in agg_pattern.findall(sql)))
+
+    def _extract_ctes(self, sql: str) -> List[str]:
+        """Extract CTE names"""
+        cte_pattern = re.compile(
+            r'\bWITH\s+(\w+)\s+AS\s*\(|,\s*(\w+)\s+AS\s*\(',
+            re.IGNORECASE
+        )
+        ctes = []
+        for match in cte_pattern.finditer(sql):
+            cte_name = match.group(1) or match.group(2)
+            if cte_name:
+                ctes.append(cte_name)
+        return ctes
+
+    def _extract_window_functions(self, sql: str) -> List[str]:
+        """Extract window function patterns"""
+        window_pattern = re.compile(
+            r'\b(\w+)\s*\([^)]*\)\s+OVER\s*\(',
+            re.IGNORECASE
+        )
+        return list(set(m.upper() for m in window_pattern.findall(sql)))
+
+    def _estimate_complexity(self, tables: List[str], joins: List[Dict],
+                            subqueries: int, aggregations: List[str],
+                            window_functions: List[str]) -> str:
+        """Estimate query complexity"""
+        score = 0
+
+        # Table count
+        score += len(tables) * 10
+
+        # Join count and types
+        for join in joins:
+            if join['type'] in ('CROSS', 'FULL OUTER'):
+                score += 30
+            elif join['type'] in ('LEFT OUTER', 'RIGHT OUTER'):
+                score += 20
+            else:
+                score += 15
+
+        # Subqueries
+        score += subqueries * 25
+
+        # Aggregations
+        score += len(aggregations) * 5
+
+        # Window functions
+        score += len(window_functions) * 15
+
+        if score < 30:
+            return 'low'
+        elif score < 60:
+            return 'medium'
+        elif score < 100:
+            return 'high'
+        else:
+            return 'very_high'
+
+
+# =============================================================================
+# SQL Optimizer
+# =============================================================================
+
+class SQLOptimizer:
+    """Analyze SQL queries and provide optimization recommendations"""
+
+    def analyze(self, query_info: SQLQueryInfo, sql: str) -> List[OptimizationRecommendation]:
+        """Analyze a SQL query and generate optimization recommendations"""
+        recommendations = []
+
+        # Check for SELECT *
+        if '*' in query_info.columns:
+            recommendations.append(self._recommend_explicit_columns())
+
+        # Check for missing WHERE clause on large tables
+        if not query_info.where_conditions and query_info.tables:
+            recommendations.append(self._recommend_add_filters())
+
+        # Check for inefficient joins
+        join_recs = self._analyze_joins(query_info)
+        recommendations.extend(join_recs)
+
+        # Check for DISTINCT usage
+        if query_info.distinct:
+            recommendations.append(self._recommend_distinct_alternative())
+
+        # Check for ORDER BY without LIMIT
+        if query_info.order_by and not query_info.limit:
+            recommendations.append(self._recommend_add_limit())
+
+        # Check for subquery optimization
+        if query_info.subqueries > 0:
+            recommendations.append(self._recommend_cte_conversion())
+
+        # Check for index opportunities
+        index_recs = self._analyze_index_opportunities(query_info)
+        recommendations.extend(index_recs)
+
+        # Check for partition pruning
+        partition_recs = self._analyze_partition_pruning(query_info, sql)
+        recommendations.extend(partition_recs)
+
+        # Check for aggregation optimization
+        if query_info.aggregations and query_info.group_by:
+            agg_recs = self._analyze_aggregation(query_info)
+            recommendations.extend(agg_recs)
+
+        # Sort by priority
+        recommendations.sort(key=lambda r: r.priority)
+
+        return recommendations
+
+    def _recommend_explicit_columns(self) -> OptimizationRecommendation:
+        return OptimizationRecommendation(
+            category="query_structure",
+            severity="medium",
+            title="Avoid SELECT *",
+            description="Using SELECT * retrieves all columns, increasing I/O and memory usage.",
+            current_issue="Query uses SELECT * which fetches unnecessary columns",
+            recommendation="Specify only the columns you need",
+            expected_improvement="10-50% reduction in data scanned depending on table width",
+            implementation="Replace SELECT * with SELECT col1, col2, col3 ...",
+            priority=2
+        )
+
+    def _recommend_add_filters(self) -> OptimizationRecommendation:
+        return OptimizationRecommendation(
+            category="filter",
+            severity="high",
+            title="Add WHERE Clause Filters",
+            description="Query scans entire tables without filtering, causing full table scans.",
+            current_issue="No WHERE clause filters found - full table scan required",
+            recommendation="Add appropriate WHERE conditions to filter data early",
+            expected_improvement="Up to 90%+ reduction in data processed if highly selective",
+            implementation="Add WHERE column = value or WHERE date_column >= '2024-01-01'",
+            priority=1
+        )
+
+    def _analyze_joins(self, query_info: SQLQueryInfo) -> List[OptimizationRecommendation]:
+        """Analyze joins for optimization opportunities"""
+        recommendations = []
+
+        for join in query_info.joins:
+            # Check for CROSS JOIN
+            if join['type'] == 'CROSS':
+                recommendations.append(OptimizationRecommendation(
+                    category="join",
+                    severity="critical",
+                    title="Avoid CROSS JOIN",
+                    description="CROSS JOIN creates a Cartesian product, which can explode data volume.",
+                    current_issue=f"CROSS JOIN with table {join['table']} detected",
+                    recommendation="Replace with appropriate INNER/LEFT JOIN with ON condition",
+                    expected_improvement="Exponential reduction in intermediate data",
+                    implementation=f"Convert CROSS JOIN {join['table']} to INNER JOIN {join['table']} ON ...",
+                    priority=1
+                ))
+
+            # Check for missing join condition
+            if not join.get('condition'):
+                recommendations.append(OptimizationRecommendation(
+                    category="join",
+                    severity="high",
+                    title="Missing Join Condition",
+                    description="Join without explicit ON condition may cause Cartesian product.",
+                    current_issue=f"JOIN with {join['table']} has no explicit ON condition",
+                    recommendation="Add explicit ON condition to the join",
+                    expected_improvement="Prevents accidental Cartesian products",
+                    implementation=f"Add ON {join['table']}.id = other_table.foreign_key",
+                    priority=1
+                ))
+
+        # Check for many joins
+        if len(query_info.joins) > 5:
+            recommendations.append(OptimizationRecommendation(
+                category="join",
+                severity="medium",
+                title="High Number of Joins",
+                description="Many joins can lead to complex execution plans and performance issues.",
+                current_issue=f"{len(query_info.joins)} joins detected in single query",
+                recommendation="Consider breaking into smaller queries or pre-aggregating",
+                expected_improvement="Better plan optimization and memory usage",
+                implementation="Use CTEs to materialize intermediate results, or denormalize frequently joined data",
+                priority=3
+            ))
+
+        return recommendations
+
+    def _recommend_distinct_alternative(self) -> OptimizationRecommendation:
+        return OptimizationRecommendation(
+            category="query_structure",
+            severity="medium",
+            title="Consider Alternatives to DISTINCT",
+            description="DISTINCT requires sorting/hashing all rows which can be expensive.",
+            current_issue="DISTINCT used - may indicate data quality or join issues",
+            recommendation="Review if DISTINCT is necessary or if joins produce duplicates",
+            expected_improvement="Eliminates expensive deduplication step if not needed",
+            implementation="Review join conditions, or use GROUP BY if aggregating anyway",
+            priority=3
+        )
+
+    def _recommend_add_limit(self) -> OptimizationRecommendation:
+        return OptimizationRecommendation(
+            category="query_structure",
+            severity="low",
+            title="Add LIMIT to ORDER BY",
+            description="ORDER BY without LIMIT sorts entire result set unnecessarily.",
+            current_issue="ORDER BY present without LIMIT clause",
+            recommendation="Add LIMIT if only top N rows are needed",
+            expected_improvement="Significant reduction in sorting overhead for large results",
+            implementation="Add LIMIT 100 (or appropriate number) after ORDER BY",
+            priority=4
+        )
+
+    def _recommend_cte_conversion(self) -> OptimizationRecommendation:
+        return OptimizationRecommendation(
+            category="query_structure",
+            severity="medium",
+            title="Convert Subqueries to CTEs",
+            description="Subqueries can be harder to optimize and maintain than CTEs.",
+            current_issue="Subqueries detected in the query",
+            recommendation="Convert correlated subqueries to CTEs or JOINs",
+            expected_improvement="Better query plan optimization and readability",
+            implementation="WITH subquery_name AS (SELECT ...) SELECT ... FROM main_table JOIN subquery_name",
+            priority=3
+        )
+
+    def _analyze_index_opportunities(self, query_info: SQLQueryInfo) -> List[OptimizationRecommendation]:
+        """Identify potential index opportunities"""
+        recommendations = []
+
+        # Columns in WHERE clause are index candidates
+        where_columns = set()
+        for condition in query_info.where_conditions:
+            # Extract column names from conditions
+            col_pattern = re.compile(r'\b([a-zA-Z_][a-zA-Z0-9_]*)\s*(?:=|>|<|>=|<=|<>|!=|LIKE|IN|BETWEEN)', re.IGNORECASE)
+            where_columns.update(col_pattern.findall(condition))
+
+        if where_columns:
+            recommendations.append(OptimizationRecommendation(
+                category="index",
+                severity="medium",
+                title="Consider Indexes on Filter Columns",
+                description="Columns used in WHERE clauses benefit from indexes.",
+                current_issue=f"Filter columns detected: {', '.join(where_columns)}",
+                recommendation="Create indexes on frequently filtered columns",
+                expected_improvement="Orders of magnitude faster for selective queries",
+                implementation=f"CREATE INDEX idx_name ON table ({', '.join(list(where_columns)[:3])})",
+                priority=2
+            ))
+
+        # JOIN columns are index candidates
+        join_columns = set()
+        for join in query_info.joins:
+            if join.get('condition'):
+                col_pattern = re.compile(r'\.([a-zA-Z_][a-zA-Z0-9_]*)\s*=', re.IGNORECASE)
+                join_columns.update(col_pattern.findall(join['condition']))
+
+        if join_columns:
+            recommendations.append(OptimizationRecommendation(
+                category="index",
+                severity="high",
+                title="Index Join Columns",
+                description="Join columns without indexes cause expensive full table scans.",
+                current_issue=f"Join columns detected: {', '.join(join_columns)}",
+                recommendation="Ensure indexes exist on join key columns",
+                expected_improvement="Dramatic improvement in join performance",
+                implementation=f"CREATE INDEX idx_join ON table ({list(join_columns)[0]})",
+                priority=1
+            ))
+
+        return recommendations
+
+    def _analyze_partition_pruning(self, query_info: SQLQueryInfo, sql: str) -> List[OptimizationRecommendation]:
+        """Check for partition pruning opportunities"""
+        recommendations = []
+
+        # Look for date/time columns in WHERE clause
+        date_pattern = re.compile(
+            r'\b(date|time|timestamp|created|updated|modified)_?\w*\s*(?:=|>|<|>=|<=|BETWEEN)',
+            re.IGNORECASE
+        )
+
+        if date_pattern.search(sql):
+            recommendations.append(OptimizationRecommendation(
+                category="partition",
+                severity="medium",
+                title="Leverage Partition Pruning",
+                description="Date-based filters can leverage partitioned tables for massive speedups.",
+                current_issue="Date/time filter detected - ensure table is partitioned",
+                recommendation="Partition table by date column and ensure filter format matches",
+                expected_improvement="90%+ reduction in data scanned for time-bounded queries",
+                implementation="CREATE TABLE ... PARTITION BY RANGE (date_column) or use dynamic partitioning",
+                priority=2
+            ))
+
+        return recommendations
+
+    def _analyze_aggregation(self, query_info: SQLQueryInfo) -> List[OptimizationRecommendation]:
+        """Analyze aggregation patterns"""
+        recommendations = []
+
+        # High cardinality GROUP BY warning
+        if len(query_info.group_by) > 3:
+            recommendations.append(OptimizationRecommendation(
+                category="aggregation",
+                severity="medium",
+                title="High Cardinality GROUP BY",
+                description="Grouping by many columns increases memory usage and reduces aggregation benefit.",
+                current_issue=f"GROUP BY with {len(query_info.group_by)} columns detected",
+                recommendation="Review if all group by columns are necessary",
+                expected_improvement="Reduced memory and faster aggregation",
+                implementation="Remove non-essential GROUP BY columns or pre-aggregate",
+                priority=3
+            ))
+
+        # COUNT DISTINCT optimization
+        if 'COUNT' in query_info.aggregations and query_info.distinct:
+            recommendations.append(OptimizationRecommendation(
+                category="aggregation",
+                severity="medium",
+                title="Optimize COUNT DISTINCT",
+                description="COUNT DISTINCT can be expensive for high cardinality columns.",
+                current_issue="COUNT DISTINCT pattern detected",
+                recommendation="Consider HyperLogLog approximation for very large datasets",
+                expected_improvement="Massive speedup with ~2% error tolerance",
+                implementation="Use APPROX_COUNT_DISTINCT() if available in your warehouse",
+                priority=3
+            ))
+
+        return recommendations
+
+
+# =============================================================================
+# Spark Job Analyzer
+# =============================================================================
+
+class SparkJobAnalyzer:
+    """Analyze Spark job metrics and provide optimization recommendations"""
+
+    def analyze(self, metrics: SparkJobMetrics) -> List[OptimizationRecommendation]:
+        """Analyze Spark job metrics"""
+        recommendations = []
+
+        # Check for data skew
+        if metrics.skew_ratio > 5:
+            recommendations.append(self._recommend_skew_mitigation(metrics))
+
+        # Check for excessive shuffle
+        shuffle_ratio = metrics.shuffle_write_bytes / max(metrics.input_bytes, 1)
+        if shuffle_ratio > 1.5:
+            recommendations.append(self._recommend_reduce_shuffle(metrics, shuffle_ratio))
+
+        # Check for GC overhead
+        gc_ratio = metrics.gc_time_ms / max(metrics.duration_ms, 1)
+        if gc_ratio > 0.1:
+            recommendations.append(self._recommend_memory_tuning(metrics, gc_ratio))
+
+        # Check for failed tasks
+        if metrics.failed_tasks > 0:
+            fail_ratio = metrics.failed_tasks / max(metrics.tasks, 1)
+            recommendations.append(self._recommend_failure_handling(metrics, fail_ratio))
+
+        # Check for speculative execution overhead
+        if metrics.speculative_tasks > metrics.tasks * 0.1:
+            recommendations.append(self._recommend_reduce_speculation(metrics))
+
+        # Check task count
+        if metrics.tasks > 10000:
+            recommendations.append(self._recommend_reduce_tasks(metrics))
+        elif metrics.tasks < 10 and metrics.input_bytes > 1e9:
+            recommendations.append(self._recommend_increase_parallelism(metrics))
+
+        return recommendations
+
+    def _recommend_skew_mitigation(self, metrics: SparkJobMetrics) -> OptimizationRecommendation:
+        return OptimizationRecommendation(
+            category="skew",
+            severity="critical",
+            title="Severe Data Skew Detected",
+            description=f"Skew ratio of {metrics.skew_ratio:.1f}x indicates uneven data distribution.",
+            current_issue=f"Task execution time varies by {metrics.skew_ratio:.1f}x, causing stragglers",
+            recommendation="Apply skew handling techniques to rebalance data",
+            expected_improvement="Up to 80% reduction in job time by eliminating stragglers",
+            implementation="""Options:
+1. Salting: Add random prefix to skewed keys
+   df.withColumn("salted_key", concat(col("key"), lit("_"), (rand() * 10).cast("int")))
+2. Broadcast join for small tables:
+   df1.join(broadcast(df2), "key")
+3. Adaptive Query Execution (Spark 3.0+):
+   spark.conf.set("spark.sql.adaptive.enabled", "true")
+   spark.conf.set("spark.sql.adaptive.skewJoin.enabled", "true")""",
+            priority=1
+        )
+
+    def _recommend_reduce_shuffle(self, metrics: SparkJobMetrics, ratio: float) -> OptimizationRecommendation:
+        return OptimizationRecommendation(
+            category="shuffle",
+            severity="high",
+            title="Excessive Shuffle Data",
+            description=f"Shuffle writes {ratio:.1f}x the input data size.",
+            current_issue=f"Shuffle write: {metrics.shuffle_write_bytes / 1e9:.2f} GB vs input: {metrics.input_bytes / 1e9:.2f} GB",
+            recommendation="Reduce shuffle through partitioning and early aggregation",
+            expected_improvement="Significant network I/O and storage reduction",
+            implementation="""Options:
+1. Pre-aggregate before shuffle:
+   df.groupBy("key").agg(sum("value")).repartition("key")
+2. Use map-side combining:
+   df.reduceByKey((a, b) => a + b)
+3. Optimize partition count:
+   spark.conf.set("spark.sql.shuffle.partitions", optimal_count)
+4. Use bucketing for repeated joins:
+   df.write.bucketBy(200, "key").saveAsTable("bucketed_table")""",
+            priority=1
+        )
+
+    def _recommend_memory_tuning(self, metrics: SparkJobMetrics, gc_ratio: float) -> OptimizationRecommendation:
+        return OptimizationRecommendation(
+            category="memory",
+            severity="high",
+            title="High GC Overhead",
+            description=f"GC time is {gc_ratio * 100:.1f}% of total execution time.",
+            current_issue=f"GC time: {metrics.gc_time_ms / 1000:.1f}s out of {metrics.duration_ms / 1000:.1f}s total",
+            recommendation="Tune memory settings to reduce garbage collection",
+            expected_improvement="20-50% faster execution with proper memory config",
+            implementation="""Memory tuning options:
+1. Increase executor memory:
+   --executor-memory 8g
+2. Adjust memory fractions:
+   spark.memory.fraction=0.6
+   spark.memory.storageFraction=0.5
+3. Use off-heap memory:
+   spark.memory.offHeap.enabled=true
+   spark.memory.offHeap.size=4g
+4. Reduce cached data:
+   df.unpersist() when no longer needed
+5. Use Kryo serialization:
+   spark.serializer=org.apache.spark.serializer.KryoSerializer""",
+            priority=2
+        )
+
+    def _recommend_failure_handling(self, metrics: SparkJobMetrics, fail_ratio: float) -> OptimizationRecommendation:
+        return OptimizationRecommendation(
+            category="reliability",
+            severity="high" if fail_ratio > 0.1 else "medium",
+            title="Task Failures Detected",
+            description=f"{metrics.failed_tasks} tasks failed ({fail_ratio * 100:.1f}% failure rate).",
+            current_issue="Task failures increase job time and resource usage due to retries",
+            recommendation="Investigate failure causes and add resilience",
+            expected_improvement="Reduced retries and more predictable job times",
+            implementation="""Failure handling options:
+1. Check executor logs for OOM:
+   spark.executor.memoryOverhead=2g
+2. Handle data issues:
+   df.filter(col("value").isNotNull())
+3. Increase task retries:
+   spark.task.maxFailures=4
+4. Add checkpointing for long jobs:
+   df.checkpoint()
+5. Check for network timeouts:
+   spark.network.timeout=300s""",
+            priority=1
+        )
+
+    def _recommend_reduce_speculation(self, metrics: SparkJobMetrics) -> OptimizationRecommendation:
+        return OptimizationRecommendation(
+            category="execution",
+            severity="medium",
+            title="High Speculative Execution",
+            description=f"{metrics.speculative_tasks} speculative tasks launched.",
+            current_issue="Excessive speculation wastes resources and indicates underlying issues",
+            recommendation="Address root cause of slow tasks instead of speculation",
+            expected_improvement="Better resource utilization",
+            implementation="""Options:
+1. Disable speculation if not needed:
+   spark.speculation=false
+2. Or tune speculation settings:
+   spark.speculation.multiplier=1.5
+   spark.speculation.quantile=0.9
+3. Fix underlying skew/memory issues first""",
+            priority=3
+        )
+
+    def _recommend_reduce_tasks(self, metrics: SparkJobMetrics) -> OptimizationRecommendation:
+        return OptimizationRecommendation(
+            category="parallelism",
+            severity="medium",
+            title="Too Many Tasks",
+            description=f"{metrics.tasks} tasks may cause excessive scheduling overhead.",
+            current_issue="Very high task count increases driver overhead",
+            recommendation="Reduce partition count for better efficiency",
+            expected_improvement="Reduced scheduling overhead and driver memory usage",
+            implementation=f"""
+1. Reduce shuffle partitions:
+   spark.sql.shuffle.partitions={max(200, metrics.tasks // 10)}
+2. Coalesce partitions:
+   df.coalesce({max(200, metrics.tasks // 10)})
+3. Use adaptive partitioning (Spark 3.0+):
+   spark.sql.adaptive.enabled=true""",
+            priority=3
+        )
+
+    def _recommend_increase_parallelism(self, metrics: SparkJobMetrics) -> OptimizationRecommendation:
+        recommended_partitions = max(200, int(metrics.input_bytes / (128 * 1e6)))  # 128MB per partition
+        return OptimizationRecommendation(
+            category="parallelism",
+            severity="high",
+            title="Low Parallelism",
+            description=f"Only {metrics.tasks} tasks for {metrics.input_bytes / 1e9:.2f} GB of data.",
+            current_issue="Under-utilization of cluster resources",
+            recommendation="Increase parallelism to better utilize cluster",
+            expected_improvement="Linear speedup with added parallelism",
+            implementation=f"""
+1. Increase shuffle partitions:
+   spark.sql.shuffle.partitions={recommended_partitions}
+2. Repartition input:
+   df.repartition({recommended_partitions})
+3. Adjust default parallelism:
+   spark.default.parallelism={recommended_partitions}""",
+            priority=2
+        )
+
+
+# =============================================================================
+# Partition Strategy Advisor
+# =============================================================================
+
+class PartitionAdvisor:
+    """Recommend partitioning strategies based on data characteristics"""
+
+    def recommend(self, data_stats: Dict) -> List[PartitionStrategy]:
+        """Generate partition recommendations from data statistics"""
+        recommendations = []
+
+        columns = data_stats.get('columns', {})
+        total_size_bytes = data_stats.get('total_size_bytes', 0)
+        row_count = data_stats.get('row_count', 0)
+
+        for col_name, col_stats in columns.items():
+            strategy = self._evaluate_column(col_name, col_stats, total_size_bytes, row_count)
+            if strategy:
+                recommendations.append(strategy)
+
+        # Sort by partition effectiveness
+        recommendations.sort(key=lambda s: s.partition_size_mb)
+
+        return recommendations[:3]  # Top 3 recommendations
+
+    def _evaluate_column(self, col_name: str, col_stats: Dict,
+                        total_size_bytes: int, row_count: int) -> Optional[PartitionStrategy]:
+        """Evaluate a column for partitioning potential"""
+        cardinality = col_stats.get('cardinality', 0)
+        data_type = col_stats.get('data_type', 'string')
+        null_percentage = col_stats.get('null_percentage', 0)
+
+        # Skip high-null columns
+        if null_percentage > 20:
+            return None
+
+        # Date/timestamp columns are ideal for range partitioning
+        if data_type in ('date', 'timestamp', 'datetime'):
+            return self._recommend_date_partition(col_name, col_stats, total_size_bytes, row_count)
+
+        # Low cardinality columns are good for list partitioning
+        if cardinality and cardinality <= 100:
+            return self._recommend_list_partition(col_name, col_stats, total_size_bytes, cardinality)
+
+        # Medium cardinality columns can use hash partitioning
+        if cardinality and 100 < cardinality <= 10000:
+            return self._recommend_hash_partition(col_name, col_stats, total_size_bytes)
+
+        return None
+
+    def _recommend_date_partition(self, col_name: str, col_stats: Dict,
+                                  total_size_bytes: int, row_count: int) -> PartitionStrategy:
+        # Estimate daily partition size (assume 365 days of data)
+        estimated_days = 365
+        partition_size_mb = (total_size_bytes / estimated_days) / (1024 * 1024)
+
+        return PartitionStrategy(
+            column=col_name,
+            partition_type="range",
+            num_partitions=None,  # Dynamic based on date range
+            partition_size_mb=partition_size_mb,
+            reasoning=f"Date column '{col_name}' is ideal for range partitioning. "
+                     f"Estimated daily partition size: {partition_size_mb:.1f} MB",
+            implementation=f"""
+-- BigQuery
+CREATE TABLE table_name
+PARTITION BY DATE({col_name})
+AS SELECT * FROM source_table;
+
+-- Snowflake
+CREATE TABLE table_name
+CLUSTER BY (DATE_TRUNC('DAY', {col_name}));
+
+-- Spark/Hive
+df.write.partitionBy("{col_name}").parquet("path")
+
+-- PostgreSQL
+CREATE TABLE table_name (...)
+PARTITION BY RANGE ({col_name});"""
+        )
+
+    def _recommend_list_partition(self, col_name: str, col_stats: Dict,
+                                  total_size_bytes: int, cardinality: int) -> PartitionStrategy:
+        partition_size_mb = (total_size_bytes / cardinality) / (1024 * 1024)
+
+        return PartitionStrategy(
+            column=col_name,
+            partition_type="list",
+            num_partitions=cardinality,
+            partition_size_mb=partition_size_mb,
+            reasoning=f"Column '{col_name}' has {cardinality} distinct values - ideal for list partitioning. "
+                     f"Estimated partition size: {partition_size_mb:.1f} MB",
+            implementation=f"""
+-- Spark/Hive
+df.write.partitionBy("{col_name}").parquet("path")
+
+-- PostgreSQL
+CREATE TABLE table_name (...)
+PARTITION BY LIST ({col_name});
+
+-- Note: List partitioning works best with stable, low-cardinality values"""
+        )
+
+    def _recommend_hash_partition(self, col_name: str, col_stats: Dict,
+                                  total_size_bytes: int) -> PartitionStrategy:
+        # Target ~128MB partitions
+        target_partition_size = 128 * 1024 * 1024
+        num_partitions = max(1, int(total_size_bytes / target_partition_size))
+
+        # Round to power of 2 for better distribution
+        num_partitions = 2 ** int(math.log2(num_partitions) + 0.5)
+        partition_size_mb = (total_size_bytes / num_partitions) / (1024 * 1024)
+
+        return PartitionStrategy(
+            column=col_name,
+            partition_type="hash",
+            num_partitions=num_partitions,
+            partition_size_mb=partition_size_mb,
+            reasoning=f"Column '{col_name}' has medium cardinality - hash partitioning provides even distribution. "
+                     f"Recommended {num_partitions} partitions (~{partition_size_mb:.1f} MB each)",
+            implementation=f"""
+-- Spark
+df.repartition({num_partitions}, col("{col_name}"))
+
+-- PostgreSQL
+CREATE TABLE table_name (...)
+PARTITION BY HASH ({col_name});
+
+-- Snowflake (clustering)
+ALTER TABLE table_name CLUSTER BY ({col_name});"""
+        )
+
+
+# =============================================================================
+# Cost Estimator
+# =============================================================================
+
+class CostEstimator:
+    """Estimate query costs for cloud data warehouses"""
+
+    # Pricing (approximate, varies by region and contract)
+    PRICING = {
+        'snowflake': {
+            'compute_per_credit': 2.00,  # USD per credit
+            'credits_per_hour': {
+                'x-small': 1,
+                'small': 2,
+                'medium': 4,
+                'large': 8,
+                'x-large': 16,
+            },
+            'storage_per_tb_month': 23.00,
+        },
+        'bigquery': {
+            'on_demand_per_tb': 5.00,  # USD per TB scanned
+            'storage_per_tb_month': 20.00,
+            'streaming_insert_per_gb': 0.01,
+        },
+        'redshift': {
+            'dc2_large_per_hour': 0.25,
+            'ra3_xlarge_per_hour': 1.086,
+            'storage_per_gb_month': 0.024,
+        },
+        'databricks': {
+            'dbu_per_hour_sql': 0.22,
+            'dbu_per_hour_jobs': 0.15,
+        }
+    }
+
+    def estimate(self, query_info: SQLQueryInfo, warehouse: str,
+                data_stats: Optional[Dict] = None) -> CostEstimate:
+        """Estimate query cost"""
+        warehouse = warehouse.lower()
+
+        if warehouse not in self.PRICING:
+            raise ValueError(f"Unknown warehouse: {warehouse}. Supported: {list(self.PRICING.keys())}")
+
+        # Estimate data scanned
+        data_scanned_bytes = self._estimate_data_scanned(query_info, data_stats)
+        data_scanned_tb = data_scanned_bytes / (1024 ** 4)
+
+        if warehouse == 'bigquery':
+            return self._estimate_bigquery(query_info, data_scanned_tb, data_stats)
+        elif warehouse == 'snowflake':
+            return self._estimate_snowflake(query_info, data_scanned_tb, data_stats)
+        elif warehouse == 'redshift':
+            return self._estimate_redshift(query_info, data_scanned_tb, data_stats)
+        elif warehouse == 'databricks':
+            return self._estimate_databricks(query_info, data_scanned_tb, data_stats)
+
+    def _estimate_data_scanned(self, query_info: SQLQueryInfo,
+                               data_stats: Optional[Dict]) -> int:
+        """Estimate bytes of data that will be scanned"""
+        if data_stats and 'total_size_bytes' in data_stats:
+            base_size = data_stats['total_size_bytes']
+        else:
+            # Default assumption: 1GB per table
+            base_size = len(query_info.tables) * 1e9
+
+        # Adjust for filters
+        filter_factor = 1.0
+        if query_info.where_conditions:
+            # Assume each filter reduces data by 50% (very rough)
+            filter_factor = 0.5 ** min(len(query_info.where_conditions), 3)
+
+        # Adjust for column projection
+        if '*' not in query_info.columns and query_info.columns:
+            # Assume selecting specific columns reduces scan by 50%
+            filter_factor *= 0.5
+
+        return int(base_size * filter_factor)
+
+    def _estimate_bigquery(self, query_info: SQLQueryInfo,
+                          data_scanned_tb: float, data_stats: Optional[Dict]) -> CostEstimate:
+        pricing = self.PRICING['bigquery']
+
+        compute_cost = data_scanned_tb * pricing['on_demand_per_tb']
+
+        # Minimum billing of 10MB
+        if data_scanned_tb < 10 / (1024 ** 2):
+            compute_cost = 10 / (1024 ** 2) * pricing['on_demand_per_tb']
+
+        return CostEstimate(
+            warehouse='BigQuery',
+            compute_cost=compute_cost,
+            storage_cost=0,  # Storage cost separate
+            data_transfer_cost=0,
+            total_cost=compute_cost,
+            assumptions=[
+                f"Estimated {data_scanned_tb * 1024:.2f} GB data scanned",
+                "Using on-demand pricing ($5/TB)",
+                "Assumes no slot reservations",
+                "Actual cost depends on partitioning and clustering"
+            ]
+        )
+
+    def _estimate_snowflake(self, query_info: SQLQueryInfo,
+                           data_scanned_tb: float, data_stats: Optional[Dict]) -> CostEstimate:
+        pricing = self.PRICING['snowflake']
+
+        # Estimate warehouse size and time
+        complexity_to_size = {
+            'low': 'x-small',
+            'medium': 'small',
+            'high': 'medium',
+            'very_high': 'large'
+        }
+        warehouse_size = complexity_to_size.get(query_info.estimated_complexity, 'small')
+        credits_per_hour = pricing['credits_per_hour'][warehouse_size]
+
+        # Estimate runtime (very rough)
+        estimated_seconds = max(1, data_scanned_tb * 1024 * 10)  # 10 seconds per GB
+        estimated_hours = estimated_seconds / 3600
+
+        credits_used = credits_per_hour * estimated_hours
+        compute_cost = credits_used * pricing['compute_per_credit']
+
+        # Minimum 1 minute billing
+        min_cost = (credits_per_hour / 60) * pricing['compute_per_credit']
+        compute_cost = max(compute_cost, min_cost)
+
+        return CostEstimate(
+            warehouse='Snowflake',
+            compute_cost=compute_cost,
+            storage_cost=0,
+            data_transfer_cost=0,
+            total_cost=compute_cost,
+            assumptions=[
+                f"Warehouse size: {warehouse_size}",
+                f"Estimated runtime: {estimated_seconds:.1f} seconds",
+                f"Credits used: {credits_used:.4f}",
+                "Minimum 1-minute billing applies",
+                "Actual cost depends on warehouse auto-suspend settings"
+            ]
+        )
+
+    def _estimate_redshift(self, query_info: SQLQueryInfo,
+                          data_scanned_tb: float, data_stats: Optional[Dict]) -> CostEstimate:
+        pricing = self.PRICING['redshift']
+
+        # Assume RA3 xl node type
+        hourly_rate = pricing['ra3_xlarge_per_hour']
+
+        # Estimate runtime
+        estimated_seconds = max(1, data_scanned_tb * 1024 * 15)  # 15 seconds per GB
+        estimated_hours = estimated_seconds / 3600
+
+        compute_cost = hourly_rate * estimated_hours
+
+        return CostEstimate(
+            warehouse='Redshift',
+            compute_cost=compute_cost,
+            storage_cost=0,
+            data_transfer_cost=0,
+            total_cost=compute_cost,
+            assumptions=[
+                f"Using RA3.xlplus node type",
+                f"Estimated runtime: {estimated_seconds:.1f} seconds",
+                "Assumes dedicated cluster (not serverless)",
+                "Actual cost depends on cluster configuration"
+            ]
+        )
+
+    def _estimate_databricks(self, query_info: SQLQueryInfo,
+                            data_scanned_tb: float, data_stats: Optional[Dict]) -> CostEstimate:
+        pricing = self.PRICING['databricks']
+
+        # Estimate DBUs
+        estimated_seconds = max(1, data_scanned_tb * 1024 * 12)
+        estimated_hours = estimated_seconds / 3600
+
+        dbu_cost = pricing['dbu_per_hour_sql'] * estimated_hours
+
+        return CostEstimate(
+            warehouse='Databricks',
+            compute_cost=dbu_cost,
+            storage_cost=0,
+            data_transfer_cost=0,
+            total_cost=dbu_cost,
+            assumptions=[
+                f"Using SQL warehouse",
+                f"Estimated runtime: {estimated_seconds:.1f} seconds",
+                "DBU rate may vary by workspace tier",
+                "Does not include underlying cloud costs"
+            ]
+        )
+
+
+# =============================================================================
+# Report Generator
+# =============================================================================
+
+class ReportGenerator:
+    """Generate optimization reports"""
+
+    def generate_text_report(self, query_info: SQLQueryInfo,
+                            recommendations: List[OptimizationRecommendation],
+                            cost_estimate: Optional[CostEstimate] = None) -> str:
+        """Generate a text report"""
+        lines = []
+        lines.append("=" * 80)
+        lines.append("ETL PERFORMANCE OPTIMIZATION REPORT")
+        lines.append("=" * 80)
+        lines.append(f"\nGenerated: {datetime.now().isoformat()}")
+
+        # Query summary
+        lines.append("\n" + "-" * 40)
+        lines.append("QUERY ANALYSIS")
+        lines.append("-" * 40)
+        lines.append(f"Query Type: {query_info.query_type}")
+        lines.append(f"Tables: {', '.join(query_info.tables) or 'None'}")
+        lines.append(f"Joins: {len(query_info.joins)}")
+        lines.append(f"Subqueries: {query_info.subqueries}")
+        lines.append(f"Aggregations: {', '.join(query_info.aggregations) or 'None'}")
+        lines.append(f"Window Functions: {', '.join(query_info.window_functions) or 'None'}")
+        lines.append(f"Complexity: {query_info.estimated_complexity.upper()}")
+
+        # Cost estimate
+        if cost_estimate:
+            lines.append("\n" + "-" * 40)
+            lines.append("COST ESTIMATE")
+            lines.append("-" * 40)
+            lines.append(f"Warehouse: {cost_estimate.warehouse}")
+            lines.append(f"Estimated Cost: ${cost_estimate.total_cost:.4f} {cost_estimate.currency}")
+            lines.append("Assumptions:")
+            for assumption in cost_estimate.assumptions:
+                lines.append(f"  - {assumption}")
+
+        # Recommendations
+        if recommendations:
+            lines.append("\n" + "-" * 40)
+            lines.append(f"OPTIMIZATION RECOMMENDATIONS ({len(recommendations)} found)")
+            lines.append("-" * 40)
+
+            for i, rec in enumerate(recommendations, 1):
+                severity_icon = {
+                    'critical': '🔴',
+                    'high': '🟠',
+                    'medium': '🟡',
+                    'low': '🟢'
+                }.get(rec.severity, '⚪')
+
+                lines.append(f"\n{i}. {severity_icon} [{rec.severity.upper()}] {rec.title}")
+                lines.append(f"   Category: {rec.category}")
+                lines.append(f"   Issue: {rec.current_issue}")
+                lines.append(f"   Recommendation: {rec.recommendation}")
+                lines.append(f"   Expected Improvement: {rec.expected_improvement}")
+                lines.append(f"\n   Implementation:")
+                for impl_line in rec.implementation.strip().split('\n'):
+                    lines.append(f"   {impl_line}")
+        else:
+            lines.append("\n✅ No optimization issues detected")
+
+        lines.append("\n" + "=" * 80)
+
+        return "\n".join(lines)
+
+    def generate_json_report(self, query_info: SQLQueryInfo,
+                            recommendations: List[OptimizationRecommendation],
+                            cost_estimate: Optional[CostEstimate] = None) -> Dict:
+        """Generate a JSON report"""
+        return {
+            "report_type": "etl_performance_optimization",
+            "generated_at": datetime.now().isoformat(),
+            "query_analysis": {
+                "query_type": query_info.query_type,
+                "tables": query_info.tables,
+                "joins": query_info.joins,
+                "subqueries": query_info.subqueries,
+                "aggregations": query_info.aggregations,
+                "window_functions": query_info.window_functions,
+                "complexity": query_info.estimated_complexity
+            },
+            "cost_estimate": asdict(cost_estimate) if cost_estimate else None,
+            "recommendations": [asdict(r) for r in recommendations],
+            "summary": {
+                "total_recommendations": len(recommendations),
+                "critical": sum(1 for r in recommendations if r.severity == "critical"),
+                "high": sum(1 for r in recommendations if r.severity == "high"),
+                "medium": sum(1 for r in recommendations if r.severity == "medium"),
+                "low": sum(1 for r in recommendations if r.severity == "low")
+            }
+        }
+
+
+# =============================================================================
+# CLI Commands
+# =============================================================================
+
+def cmd_analyze_sql(args):
+    """Analyze SQL query for optimization opportunities"""
+    # Load SQL
+    sql_path = Path(args.input)
+    if sql_path.exists():
+        with open(sql_path, 'r') as f:
+            sql = f.read()
+    else:
+        sql = args.input  # Treat as inline SQL
+
+    # Parse and analyze
+    parser = SQLParser()
+    query_info = parser.parse(sql)
+
+    optimizer = SQLOptimizer()
+    recommendations = optimizer.analyze(query_info, sql)
+
+    # Cost estimate if warehouse specified
+    cost_estimate = None
+    if args.warehouse:
+        estimator = CostEstimator()
+        data_stats = None
+        if args.stats:
+            with open(args.stats, 'r') as f:
+                data_stats = json.load(f)
+        cost_estimate = estimator.estimate(query_info, args.warehouse, data_stats)
+
+    # Generate report
+    reporter = ReportGenerator()
+
+    if args.json:
+        report = reporter.generate_json_report(query_info, recommendations, cost_estimate)
+        output = json.dumps(report, indent=2)
+    else:
+        output = reporter.generate_text_report(query_info, recommendations, cost_estimate)
+
+    if args.output:
+        with open(args.output, 'w') as f:
+            f.write(output)
+        logger.info(f"Report saved to {args.output}")
+    else:
+        print(output)
+
+
+def cmd_analyze_spark(args):
+    """Analyze Spark job metrics"""
+    with open(args.input, 'r') as f:
+        metrics_data = json.load(f)
+
+    # Handle both single job and array of jobs
+    if isinstance(metrics_data, list):
+        jobs = metrics_data
+    else:
+        jobs = [metrics_data]
+
+    all_recommendations = []
+    analyzer = SparkJobAnalyzer()
+
+    for job_data in jobs:
+        metrics = SparkJobMetrics(
+            job_id=job_data.get('jobId', 'unknown'),
+            duration_ms=job_data.get('duration', 0),
+            stages=job_data.get('numStages', 0),
+            tasks=job_data.get('numTasks', 0),
+            shuffle_read_bytes=job_data.get('shuffleReadBytes', 0),
+            shuffle_write_bytes=job_data.get('shuffleWriteBytes', 0),
+            input_bytes=job_data.get('inputBytes', 0),
+            output_bytes=job_data.get('outputBytes', 0),
+            peak_memory_bytes=job_data.get('peakMemoryBytes', 0),
+            gc_time_ms=job_data.get('gcTime', 0),
+            failed_tasks=job_data.get('failedTasks', 0),
+            speculative_tasks=job_data.get('speculativeTasks', 0),
+            skew_ratio=job_data.get('skewRatio', 1.0)
+        )
+
+        recommendations = analyzer.analyze(metrics)
+        all_recommendations.extend(recommendations)
+
+    # Deduplicate similar recommendations
+    unique_recs = []
+    seen_titles = set()
+    for rec in all_recommendations:
+        if rec.title not in seen_titles:
+            unique_recs.append(rec)
+            seen_titles.add(rec.title)
+
+    # Output
+    if args.json:
+        output = json.dumps([asdict(r) for r in unique_recs], indent=2)
+    else:
+        lines = []
+        lines.append("=" * 60)
+        lines.append("SPARK JOB OPTIMIZATION REPORT")
+        lines.append("=" * 60)
+        lines.append(f"\nJobs Analyzed: {len(jobs)}")
+        lines.append(f"Recommendations: {len(unique_recs)}")
+
+        for i, rec in enumerate(unique_recs, 1):
+            lines.append(f"\n{i}. [{rec.severity.upper()}] {rec.title}")
+            lines.append(f"   {rec.description}")
+            lines.append(f"   Implementation: {rec.implementation[:200]}...")
+
+        output = "\n".join(lines)
+
+    if args.output:
+        with open(args.output, 'w') as f:
+            f.write(output)
+    else:
+        print(output)
+
+
+def cmd_optimize_partition(args):
+    """Recommend partition strategies"""
+    with open(args.input, 'r') as f:
+        data_stats = json.load(f)
+
+    advisor = PartitionAdvisor()
+    strategies = advisor.recommend(data_stats)
+
+    if args.json:
+        output = json.dumps([asdict(s) for s in strategies], indent=2)
+    else:
+        lines = []
+        lines.append("=" * 60)
+        lines.append("PARTITION STRATEGY RECOMMENDATIONS")
+        lines.append("=" * 60)
+
+        if not strategies:
+            lines.append("\nNo partition recommendations based on provided data statistics.")
+        else:
+            for i, strategy in enumerate(strategies, 1):
+                lines.append(f"\n{i}. Partition by: {strategy.column}")
+                lines.append(f"   Type: {strategy.partition_type}")
+                if strategy.num_partitions:
+                    lines.append(f"   Partitions: {strategy.num_partitions}")
+                lines.append(f"   Estimated size: {strategy.partition_size_mb:.1f} MB per partition")
+                lines.append(f"   Reasoning: {strategy.reasoning}")
+                lines.append(f"\n   Implementation:")
+                for impl_line in strategy.implementation.strip().split('\n'):
+                    lines.append(f"   {impl_line}")
+
+        output = "\n".join(lines)
+
+    if args.output:
+        with open(args.output, 'w') as f:
+            f.write(output)
+    else:
+        print(output)
+
+
+def cmd_estimate_cost(args):
+    """Estimate query cost"""
+    # Load SQL
+    sql_path = Path(args.input)
+    if sql_path.exists():
+        with open(sql_path, 'r') as f:
+            sql = f.read()
+    else:
+        sql = args.input
+
+    # Parse
+    parser = SQLParser()
+    query_info = parser.parse(sql)
+
+    # Load data stats if provided
+    data_stats = None
+    if args.stats:
+        with open(args.stats, 'r') as f:
+            data_stats = json.load(f)
+
+    # Estimate cost
+    estimator = CostEstimator()
+    cost = estimator.estimate(query_info, args.warehouse, data_stats)
+
+    if args.json:
+        output = json.dumps(asdict(cost), indent=2)
+    else:
+        lines = []
+        lines.append(f"Cost Estimate for {cost.warehouse}")
+        lines.append("=" * 40)
+        lines.append(f"Compute Cost:      ${cost.compute_cost:.4f}")
+        lines.append(f"Storage Cost:      ${cost.storage_cost:.4f}")
+        lines.append(f"Data Transfer:     ${cost.data_transfer_cost:.4f}")
+        lines.append("-" * 40)
+        lines.append(f"Total:             ${cost.total_cost:.4f} {cost.currency}")
+        lines.append("\nAssumptions:")
+        for assumption in cost.assumptions:
+            lines.append(f"  - {assumption}")
+
+        output = "\n".join(lines)
+
+    if args.output:
+        with open(args.output, 'w') as f:
+            f.write(output)
+    else:
+        print(output)
+
+
+def cmd_generate_template(args):
+    """Generate template files"""
+    templates = {
+        'data_stats': {
+            "total_size_bytes": 10737418240,
+            "row_count": 10000000,
+            "columns": {
+                "id": {
+                    "data_type": "integer",
+                    "cardinality": 10000000,
+                    "null_percentage": 0
+                },
+                "created_at": {
+                    "data_type": "timestamp",
+                    "cardinality": 1000000,
+                    "null_percentage": 0
+                },
+                "category": {
+                    "data_type": "string",
+                    "cardinality": 50,
+                    "null_percentage": 2
+                },
+                "amount": {
+                    "data_type": "float",
+                    "cardinality": 100000,
+                    "null_percentage": 5
+                }
+            }
+        },
+        'spark_metrics': {
+            "jobId": "job_12345",
+            "duration": 300000,
+            "numStages": 5,
+            "numTasks": 200,
+            "shuffleReadBytes": 5368709120,
+            "shuffleWriteBytes": 2147483648,
+            "inputBytes": 10737418240,
+            "outputBytes": 1073741824,
+            "peakMemoryBytes": 4294967296,
+            "gcTime": 15000,
+            "failedTasks": 2,
+            "speculativeTasks": 5,
+            "skewRatio": 3.5
+        }
+    }
+
+    if args.template not in templates:
+        logger.error(f"Unknown template: {args.template}. Available: {list(templates.keys())}")
+        sys.exit(1)
+
+    output = json.dumps(templates[args.template], indent=2)
+
+    if args.output:
+        with open(args.output, 'w') as f:
+            f.write(output)
+        logger.info(f"Template saved to {args.output}")
+    else:
+        print(output)
+
+
+def main():
+    """Main entry point"""
+    parser = argparse.ArgumentParser(
+        description="ETL Performance Optimizer - Analyze and optimize data pipelines",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Analyze SQL query
+  python etl_performance_optimizer.py analyze-sql query.sql
+
+  # Analyze with cost estimate
+  python etl_performance_optimizer.py analyze-sql query.sql --warehouse bigquery
+
+  # Analyze Spark job metrics
+  python etl_performance_optimizer.py analyze-spark spark-history.json
+
+  # Get partition recommendations
+  python etl_performance_optimizer.py optimize-partition data_stats.json
+
+  # Estimate query cost
+  python etl_performance_optimizer.py estimate-cost query.sql --warehouse snowflake
+
+  # Generate template files
+  python etl_performance_optimizer.py template data_stats --output stats.json
+        """
+    )
+
+    parser.add_argument('--verbose', '-v', action='store_true', help='Verbose output')
+
+    subparsers = parser.add_subparsers(dest='command', help='Command to run')
+
+    # Analyze SQL command
+    sql_parser = subparsers.add_parser('analyze-sql', help='Analyze SQL query')
+    sql_parser.add_argument('input', help='SQL file or inline query')
+    sql_parser.add_argument('--warehouse', '-w', choices=['bigquery', 'snowflake', 'redshift', 'databricks'],
+                          help='Warehouse for cost estimation')
+    sql_parser.add_argument('--stats', '-s', help='Data statistics JSON file')
+    sql_parser.add_argument('--output', '-o', help='Output file')
+    sql_parser.add_argument('--json', action='store_true', help='Output as JSON')
+    sql_parser.set_defaults(func=cmd_analyze_sql)
+
+    # Analyze Spark command
+    spark_parser = subparsers.add_parser('analyze-spark', help='Analyze Spark job metrics')
+    spark_parser.add_argument('input', help='Spark metrics JSON file')
+    spark_parser.add_argument('--output', '-o', help='Output file')
+    spark_parser.add_argument('--json', action='store_true', help='Output as JSON')
+    spark_parser.set_defaults(func=cmd_analyze_spark)
+
+    # Optimize partition command
+    partition_parser = subparsers.add_parser('optimize-partition', help='Recommend partition strategies')
+    partition_parser.add_argument('input', help='Data statistics JSON file')
+    partition_parser.add_argument('--output', '-o', help='Output file')
+    partition_parser.add_argument('--json', action='store_true', help='Output as JSON')
+    partition_parser.set_defaults(func=cmd_optimize_partition)
+
+    # Estimate cost command
+    cost_parser = subparsers.add_parser('estimate-cost', help='Estimate query cost')
+    cost_parser.add_argument('input', help='SQL file or inline query')
+    cost_parser.add_argument('--warehouse', '-w', required=True,
+                           choices=['bigquery', 'snowflake', 'redshift', 'databricks'],
+                           help='Target warehouse')
+    cost_parser.add_argument('--stats', '-s', help='Data statistics JSON file')
+    cost_parser.add_argument('--output', '-o', help='Output file')
+    cost_parser.add_argument('--json', action='store_true', help='Output as JSON')
+    cost_parser.set_defaults(func=cmd_estimate_cost)
+
+    # Template command
+    template_parser = subparsers.add_parser('template', help='Generate template files')
+    template_parser.add_argument('template', choices=['data_stats', 'spark_metrics'],
+                                help='Template type')
+    template_parser.add_argument('--output', '-o', help='Output file')
+    template_parser.set_defaults(func=cmd_generate_template)
+
+    args = parser.parse_args()
+
+    if args.verbose:
+        logging.getLogger().setLevel(logging.DEBUG)
+
+    if not args.command:
+        parser.print_help()
+        sys.exit(1)
+
+    try:
+        args.func(args)
+    except Exception as e:
+        logger.error(f"Error: {e}")
+        if args.verbose:
+            import traceback
+            traceback.print_exc()
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()