ds-agent-cli 0.1.0

package/src/tools/advanced_insights.py

@@ -0,0 +1,578 @@
+"""
+Advanced Insights Tools
+Tools for root cause analysis, trend detection, anomaly detection, and statistical testing.
+"""
+
+import polars as pl
+import numpy as np
+import pandas as pd
+from typing import Dict, Any, List, Optional, Tuple
+from pathlib import Path
+import sys
+import os
+from scipy import stats
+from scipy.signal import find_peaks
+from sklearn.ensemble import IsolationForest
+from sklearn.cluster import KMeans
+from sklearn.preprocessing import StandardScaler
+import json
+
+# Add parent directory to path
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from ds_agent.utils.polars_helpers import load_dataframe, get_numeric_columns
+from ds_agent.utils.validation import validate_file_exists, validate_file_format
+
+
+def analyze_root_cause(file_path: str, 
+                       target_col: str,
+                       time_col: Optional[str] = None,
+                       threshold_drop: float = 0.15) -> Dict[str, Any]:
+    """
+    Perform root cause analysis to identify why a metric dropped.
+    
+    Args:
+        file_path: Path to dataset
+        target_col: Column to analyze (e.g., 'sales')
+        time_col: Optional time column for trend analysis
+        threshold_drop: Percentage drop to flag as significant (default 15%)
+        
+    Returns:
+        Dictionary with root cause insights
+    """
+    validate_file_exists(file_path)
+    df = load_dataframe(file_path)
+    
+    # Convert to pandas for easier analysis
+    df_pd = df.to_pandas()
+    
+    results = {
+        "target_column": target_col,
+        "analysis_type": "root_cause",
+        "insights": [],
+        "correlations": {},
+        "top_factors": []
+    }
+    
+    # Check if target exists
+    if target_col not in df_pd.columns:
+        return {"status": "error", "message": f"Column '{target_col}' not found"}
+    
+    # Analyze overall trend
+    target_mean = df_pd[target_col].mean()
+    target_std = df_pd[target_col].std()
+    
+    # If time column exists, analyze temporal patterns
+    if time_col and time_col in df_pd.columns:
+        try:
+            df_pd[time_col] = pd.to_datetime(df_pd[time_col])
+            df_sorted = df_pd.sort_values(time_col)
+            
+            # Calculate period-over-period changes
+            if len(df_sorted) > 10:
+                mid_point = len(df_sorted) // 2
+                first_half_mean = df_sorted[target_col].iloc[:mid_point].mean()
+                second_half_mean = df_sorted[target_col].iloc[mid_point:].mean()
+                
+                change_pct = ((second_half_mean - first_half_mean) / first_half_mean) * 100
+                
+                if abs(change_pct) > threshold_drop * 100:
+                    insight = f"📉 Significant change detected: {change_pct:+.1f}% between periods"
+                    results["insights"].append(insight)
+                    results["period_change"] = {
+                        "first_period_avg": float(first_half_mean),
+                        "second_period_avg": float(second_half_mean),
+                        "change_percentage": float(change_pct)
+                    }
+        except Exception as e:
+            results["insights"].append(f"⚠️ Could not analyze time series: {str(e)}")
+    
+    # Find correlations with target
+    numeric_cols = df_pd.select_dtypes(include=[np.number]).columns.tolist()
+    if target_col in numeric_cols:
+        numeric_cols.remove(target_col)
+    
+    if numeric_cols:
+        correlations = {}
+        for col in numeric_cols[:20]:  # Limit to top 20 for performance
+            try:
+                corr = df_pd[target_col].corr(df_pd[col])
+                if not np.isnan(corr):
+                    correlations[col] = float(corr)
+            except:
+                pass
+        
+        # Sort by absolute correlation
+        sorted_corrs = sorted(correlations.items(), key=lambda x: abs(x[1]), reverse=True)
+        results["correlations"] = dict(sorted_corrs[:10])
+        
+        # Identify top factors
+        top_factors = []
+        for col, corr in sorted_corrs[:5]:
+            if abs(corr) > 0.3:
+                direction = "positively" if corr > 0 else "negatively"
+                top_factors.append({
+                    "factor": col,
+                    "correlation": float(corr),
+                    "description": f"{col} is {direction} correlated ({corr:.3f}) with {target_col}"
+                })
+        
+        results["top_factors"] = top_factors
+        
+        if top_factors:
+            results["insights"].append(f"🔍 Found {len(top_factors)} significant factors influencing {target_col}")
+    
+    # Outlier detection in target
+    Q1 = df_pd[target_col].quantile(0.25)
+    Q3 = df_pd[target_col].quantile(0.75)
+    IQR = Q3 - Q1
+    outliers = df_pd[(df_pd[target_col] < Q1 - 1.5 * IQR) | (df_pd[target_col] > Q3 + 1.5 * IQR)]
+    
+    if len(outliers) > 0:
+        outlier_pct = (len(outliers) / len(df_pd)) * 100
+        results["insights"].append(f"⚠️ {len(outliers)} outliers detected ({outlier_pct:.1f}% of data)")
+        results["outlier_count"] = len(outliers)
+    
+    return results
+
+
+def detect_trends_and_seasonality(file_path: str,
+                                  value_col: str,
+                                  time_col: str,
+                                  seasonal_period: Optional[int] = None) -> Dict[str, Any]:
+    """
+    Detect trends and seasonal patterns in time series data.
+    
+    Args:
+        file_path: Path to dataset
+        value_col: Column with values to analyze
+        time_col: Column with timestamps
+        seasonal_period: Expected seasonal period (auto-detected if None)
+        
+    Returns:
+        Dictionary with trend and seasonality insights
+    """
+    validate_file_exists(file_path)
+    df = load_dataframe(file_path).to_pandas()
+    
+    results = {
+        "value_column": value_col,
+        "time_column": time_col,
+        "trend_detected": False,
+        "seasonality_detected": False,
+        "insights": []
+    }
+    
+    # Validate columns
+    if value_col not in df.columns or time_col not in df.columns:
+        return {"status": "error", "message": "Columns not found"}
+    
+    # Convert to datetime and sort
+    try:
+        df[time_col] = pd.to_datetime(df[time_col])
+        df = df.sort_values(time_col).reset_index(drop=True)
+    except:
+        return {"status": "error", "message": f"Could not parse {time_col} as datetime"}
+    
+    values = df[value_col].values
+    
+    # Trend detection using linear regression
+    X = np.arange(len(values)).reshape(-1, 1)
+    y = values
+    
+    # Simple linear regression
+    slope, intercept, r_value, p_value, std_err = stats.linregress(X.flatten(), y)
+    
+    if p_value < 0.05:  # Significant trend
+        results["trend_detected"] = True
+        results["trend_slope"] = float(slope)
+        results["trend_r_squared"] = float(r_value ** 2)
+        
+        direction = "upward" if slope > 0 else "downward"
+        results["insights"].append(f"📈 {direction.capitalize()} trend detected (slope: {slope:.4f}, R²: {r_value**2:.3f})")
+        results["trend_direction"] = direction
+    else:
+        results["insights"].append("📊 No significant trend detected")
+    
+    # Seasonality detection using autocorrelation
+    if len(values) > 20:
+        from statsmodels.tsa.stattools import acf
+        
+        try:
+            autocorr = acf(values, nlags=min(len(values)//2, 50), fft=True)
+            
+            # Find peaks in autocorrelation (excluding lag 0)
+            peaks, properties = find_peaks(autocorr[1:], height=0.3)
+            
+            if len(peaks) > 0:
+                # Most prominent peak indicates seasonal period
+                peak_lag = peaks[np.argmax(properties['peak_heights'])] + 1
+                results["seasonality_detected"] = True
+                results["seasonal_period"] = int(peak_lag)
+                results["insights"].append(f"🔄 Seasonality detected with period of {peak_lag} observations")
+            else:
+                results["insights"].append("📊 No strong seasonality pattern detected")
+        except Exception as e:
+            results["insights"].append(f"⚠️ Could not analyze seasonality: {str(e)}")
+    
+    # Calculate summary statistics
+    results["statistics"] = {
+        "mean": float(np.mean(values)),
+        "std": float(np.std(values)),
+        "min": float(np.min(values)),
+        "max": float(np.max(values)),
+        "range": float(np.max(values) - np.min(values))
+    }
+    
+    return results
+
+
+def detect_anomalies_advanced(file_path: str,
+                              columns: Optional[List[str]] = None,
+                              contamination: float = 0.1,
+                              method: str = "isolation_forest") -> Dict[str, Any]:
+    """
+    Detect anomalies with confidence scores using advanced methods.
+    
+    Args:
+        file_path: Path to dataset
+        columns: Columns to analyze (all numeric if None)
+        contamination: Expected proportion of outliers
+        method: 'isolation_forest' or 'statistical'
+        
+    Returns:
+        Dictionary with anomaly detection results
+    """
+    validate_file_exists(file_path)
+    df = load_dataframe(file_path)
+    df_pd = df.to_pandas()
+    
+    # Select numeric columns
+    if columns is None:
+        numeric_cols = df_pd.select_dtypes(include=[np.number]).columns.tolist()
+    else:
+        numeric_cols = [c for c in columns if c in df_pd.columns]
+    
+    if not numeric_cols:
+        return {"status": "error", "message": "No numeric columns found"}
+    
+    X = df_pd[numeric_cols].fillna(df_pd[numeric_cols].mean())
+    
+    results = {
+        "method": method,
+        "columns_analyzed": numeric_cols,
+        "total_rows": len(X),
+        "anomaly_indices": [],
+        "anomaly_scores": []
+    }
+    
+    if method == "isolation_forest":
+        # Isolation Forest
+        clf = IsolationForest(contamination=contamination, random_state=42)
+        predictions = clf.fit_predict(X)
+        scores = clf.score_samples(X)
+        
+        anomaly_mask = predictions == -1
+        results["anomalies_detected"] = int(anomaly_mask.sum())
+        results["anomaly_percentage"] = float((anomaly_mask.sum() / len(X)) * 100)
+        results["anomaly_indices"] = np.where(anomaly_mask)[0].tolist()
+        results["anomaly_scores"] = scores[anomaly_mask].tolist()
+        
+        results["insights"] = [
+            f"🔍 Detected {results['anomalies_detected']} anomalies ({results['anomaly_percentage']:.2f}% of data)",
+            f"📊 Using Isolation Forest with contamination={contamination}"
+        ]
+    
+    else:  # Statistical method
+        # Z-score method
+        z_scores = np.abs(stats.zscore(X, nan_policy='omit'))
+        anomaly_mask = (z_scores > 3).any(axis=1)
+        
+        results["anomalies_detected"] = int(anomaly_mask.sum())
+        results["anomaly_percentage"] = float((anomaly_mask.sum() / len(X)) * 100)
+        results["anomaly_indices"] = np.where(anomaly_mask)[0].tolist()
+        
+        results["insights"] = [
+            f"🔍 Detected {results['anomalies_detected']} anomalies ({results['anomaly_percentage']:.2f}% of data)",
+            f"📊 Using statistical method (Z-score > 3)"
+        ]
+    
+    return results
+
+
+def perform_hypothesis_testing(file_path: str,
+                               group_col: str,
+                               value_col: str,
+                               test_type: str = "auto") -> Dict[str, Any]:
+    """
+    Perform statistical hypothesis testing.
+    
+    Args:
+        file_path: Path to dataset
+        group_col: Column defining groups
+        value_col: Column with values to compare
+        test_type: 't-test', 'chi-square', 'anova', or 'auto'
+        
+    Returns:
+        Dictionary with test results
+    """
+    validate_file_exists(file_path)
+    df = load_dataframe(file_path).to_pandas()
+    
+    if group_col not in df.columns or value_col not in df.columns:
+        return {"status": "error", "message": "Columns not found"}
+    
+    results = {
+        "group_column": group_col,
+        "value_column": value_col,
+        "test_type": test_type
+    }
+    
+    # Get groups
+    groups = df.groupby(group_col)[value_col].apply(list).to_dict()
+    group_names = list(groups.keys())
+    
+    if len(group_names) < 2:
+        return {"status": "error", "message": "Need at least 2 groups for comparison"}
+    
+    # Auto-detect test type
+    if test_type == "auto":
+        if len(group_names) == 2:
+            test_type = "t-test"
+        else:
+            test_type = "anova"
+    
+    # Perform test
+    if test_type == "t-test" and len(group_names) >= 2:
+        group1_data = groups[group_names[0]]
+        group2_data = groups[group_names[1]]
+        
+        statistic, p_value = stats.ttest_ind(group1_data, group2_data)
+        
+        results["test_statistic"] = float(statistic)
+        results["p_value"] = float(p_value)
+        results["significant"] = p_value < 0.05
+        results["groups_compared"] = [group_names[0], group_names[1]]
+        
+        results["interpretation"] = (
+            f"{'Significant' if p_value < 0.05 else 'No significant'} difference "
+            f"between {group_names[0]} and {group_names[1]} (p={p_value:.4f})"
+        )
+        
+        # Effect size (Cohen's d)
+        mean1, mean2 = np.mean(group1_data), np.mean(group2_data)
+        std1, std2 = np.std(group1_data), np.std(group2_data)
+        pooled_std = np.sqrt((std1**2 + std2**2) / 2)
+        cohens_d = (mean1 - mean2) / pooled_std if pooled_std > 0 else 0
+        
+        results["effect_size"] = float(cohens_d)
+        results["group_means"] = {group_names[0]: float(mean1), group_names[1]: float(mean2)}
+    
+    elif test_type == "anova":
+        group_data = [groups[g] for g in group_names]
+        statistic, p_value = stats.f_oneway(*group_data)
+        
+        results["test_statistic"] = float(statistic)
+        results["p_value"] = float(p_value)
+        results["significant"] = p_value < 0.05
+        results["groups_compared"] = group_names
+        
+        results["interpretation"] = (
+            f"{'Significant' if p_value < 0.05 else 'No significant'} difference "
+            f"among {len(group_names)} groups (p={p_value:.4f})"
+        )
+        
+        # Group means
+        results["group_means"] = {g: float(np.mean(groups[g])) for g in group_names}
+    
+    return results
+
+
+def analyze_distribution(file_path: str,
+                        column: str,
+                        tests: List[str] = ["normality", "skewness"]) -> Dict[str, Any]:
+    """
+    Analyze distribution of a column.
+    
+    Args:
+        file_path: Path to dataset
+        column: Column to analyze
+        tests: List of tests to perform
+        
+    Returns:
+        Dictionary with distribution analysis results
+    """
+    validate_file_exists(file_path)
+    df = load_dataframe(file_path).to_pandas()
+    
+    if column not in df.columns:
+        return {"status": "error", "message": f"Column '{column}' not found"}
+    
+    data = df[column].dropna()
+    
+    results = {
+        "column": column,
+        "n_values": len(data),
+        "n_missing": int(df[column].isna().sum()),
+        "tests_performed": tests,
+        "insights": []
+    }
+    
+    # Basic statistics
+    results["statistics"] = {
+        "mean": float(data.mean()),
+        "median": float(data.median()),
+        "std": float(data.std()),
+        "min": float(data.min()),
+        "max": float(data.max()),
+        "q25": float(data.quantile(0.25)),
+        "q75": float(data.quantile(0.75))
+    }
+    
+    # Normality test
+    if "normality" in tests:
+        statistic, p_value = stats.shapiro(data.sample(min(5000, len(data))))  # Limit for performance
+        results["normality_test"] = {
+            "test": "Shapiro-Wilk",
+            "statistic": float(statistic),
+            "p_value": float(p_value),
+            "is_normal": p_value > 0.05
+        }
+        
+        if p_value > 0.05:
+            results["insights"].append(f"✅ Data appears normally distributed (p={p_value:.4f})")
+        else:
+            results["insights"].append(f"⚠️ Data is NOT normally distributed (p={p_value:.4f})")
+    
+    # Skewness
+    if "skewness" in tests:
+        skewness = float(stats.skew(data))
+        kurtosis = float(stats.kurtosis(data))
+        
+        results["skewness"] = skewness
+        results["kurtosis"] = kurtosis
+        
+        if abs(skewness) < 0.5:
+            skew_desc = "approximately symmetric"
+        elif skewness > 0:
+            skew_desc = "right-skewed (positive skew)"
+        else:
+            skew_desc = "left-skewed (negative skew)"
+        
+        results["insights"].append(f"📊 Distribution is {skew_desc} (skewness={skewness:.3f})")
+    
+    return results
+
+
+def perform_segment_analysis(file_path: str,
+                             n_segments: int = 5,
+                             features: Optional[List[str]] = None,
+                             method: str = "kmeans") -> Dict[str, Any]:
+    """
+    Perform cluster-based segment analysis.
+    
+    Args:
+        file_path: Path to dataset
+        n_segments: Number of segments to create (ignored for HDBSCAN)
+        features: Features to use for clustering (all numeric if None)
+        method: Clustering method ('kmeans' or 'hdbscan')
+        
+    Returns:
+        Dictionary with segment analysis results
+    """
+    validate_file_exists(file_path)
+    df = load_dataframe(file_path).to_pandas()
+    
+    # Select features
+    if features is None:
+        features = df.select_dtypes(include=[np.number]).columns.tolist()
+    else:
+        features = [f for f in features if f in df.columns]
+    
+    if not features:
+        return {"status": "error", "message": "No numeric features found for clustering"}
+    
+    # Prepare data
+    X = df[features].fillna(df[features].mean())
+    
+    # Scale features
+    scaler = StandardScaler()
+    X_scaled = scaler.fit_transform(X)
+    
+    # Perform clustering
+    if method == "hdbscan":
+        try:
+            from sklearn.cluster import HDBSCAN as SklearnHDBSCAN
+            
+            print("🔍 Using HDBSCAN for density-based segmentation...")
+            clusterer = SklearnHDBSCAN(
+                min_cluster_size=max(5, len(X) // 50),
+                min_samples=max(3, len(X) // 100),
+                cluster_selection_method='eom'
+            )
+            labels = clusterer.fit_predict(X_scaled)
+            
+            # HDBSCAN assigns -1 to noise points
+            n_clusters = len(set(labels)) - (1 if -1 in labels else 0)
+            n_noise = int((labels == -1).sum())
+            n_segments = n_clusters
+            
+            print(f"   Found {n_clusters} clusters + {n_noise} noise points")
+            
+        except ImportError:
+            print("⚠️ HDBSCAN not available (requires scikit-learn >= 1.3). Falling back to KMeans.")
+            method = "kmeans"
+    
+    if method == "kmeans":
+        kmeans = KMeans(n_clusters=n_segments, random_state=42, n_init=10)
+        labels = kmeans.fit_predict(X_scaled)
+    
+    # Add cluster labels to dataframe
+    df['segment'] = labels
+    
+    # Analyze segments (include noise cluster -1 for HDBSCAN)
+    unique_labels = sorted(set(labels))
+    segment_profiles = []
+    for label in unique_labels:
+        segment_data = df[df['segment'] == label]
+        profile = {
+            "segment_id": int(label),
+            "label": "noise" if label == -1 else f"cluster_{label}",
+            "size": len(segment_data),
+            "percentage": float((len(segment_data) / len(df)) * 100),
+            "characteristics": {}
+        }
+        
+        # Calculate mean for each feature
+        for feat in features:
+            profile["characteristics"][feat] = {
+                "mean": float(segment_data[feat].mean()),
+                "std": float(segment_data[feat].std())
+            }
+        
+        segment_profiles.append(profile)
+    
+    results = {
+        "method": method,
+        "n_segments": n_segments,
+        "features_used": features,
+        "total_samples": len(df),
+        "segments": segment_profiles,
+        "insights": [
+            f"🎯 Created {n_segments} segments from {len(df)} samples using {method.upper()}",
+            f"📊 Used {len(features)} features for segmentation"
+        ]
+    }
+    
+    if method == "hdbscan" and n_noise > 0:
+        results["noise_points"] = n_noise
+        results["insights"].append(f"🔇 {n_noise} samples classified as noise (outliers)")
+    
+    # Find most distinctive features for each segment
+    for profile in segment_profiles:
+        if profile["segment_id"] != -1:
+            results["insights"].append(
+                f"Segment {profile['segment_id']}: {profile['size']} samples ({profile['percentage']:.1f}%)"
+            )
+    
+    return results