ds-agent-cli 0.1.0

package/src/tools/eda_reports.py

@@ -0,0 +1,284 @@
+"""
+EDA Report Generation Tools
+Generates comprehensive HTML reports using ydata-profiling.
+"""
+
+import os
+import warnings
+import io
+import contextlib
+from pathlib import Path
+from typing import Dict, Any, Optional
+
+# Suppress multiprocessing warnings from ydata-profiling cleanup
+warnings.filterwarnings("ignore", category=UserWarning, module="multiprocessing")
+warnings.filterwarnings("ignore", message=".*resource_tracker.*")
+
+
+def generate_ydata_profiling_report(
+    file_path: str,
+    output_path: str = "./outputs/reports/ydata_profile.html",
+    minimal: bool = False,
+    title: str = "Data Profiling Report",
+    quiet: bool = True
+) -> Dict[str, Any]:
+    """
+    Generate a comprehensive HTML report using ydata-profiling (formerly pandas-profiling).
+    
+    ydata-profiling provides extensive analysis including:
+    - Overview: dataset statistics, warnings, reproduction
+    - Variables: type inference, statistics, histograms, common values, missing values
+    - Interactions: scatter plots, correlations (Pearson, Spearman, Kendall, Cramér's V)
+    - Correlations: detailed correlation matrices and heatmaps
+    - Missing values: matrix, heatmap, and dendrogram
+    - Sample: first/last rows of the dataset
+    - Duplicate rows: analysis and examples
+    
+    Args:
+        file_path: Path to the dataset CSV file
+        output_path: Where to save the HTML report
+        minimal: If True, generates faster minimal report (useful for large datasets)
+        title: Title for the report
+        
+    Returns:
+        Dict with success status, report path, and statistics
+    """
+    try:
+        from ydata_profiling import ProfileReport
+        import pandas as pd
+        
+        # Read dataset (ydata-profiling requires pandas)
+        if file_path.endswith('.csv'):
+            df = pd.read_csv(file_path)
+        elif file_path.endswith('.parquet'):
+            df = pd.read_parquet(file_path)
+        else:
+            raise ValueError(f"Unsupported file format: {file_path}")
+        
+        # Auto-optimize for large datasets to prevent memory crashes
+        rows, cols = df.shape
+        file_size_mb = os.path.getsize(file_path) / (1024 * 1024)
+        
+        # Check environment: HuggingFace has 16GB, Render has 512MB
+        # Allow larger datasets on high-memory environments
+        max_rows_threshold = int(os.getenv("YDATA_MAX_ROWS", "100000"))  # Default: 100k (HF), or set to 50000 for low-mem
+        max_size_threshold = float(os.getenv("YDATA_MAX_SIZE_MB", "50"))  # Default: 50MB
+        
+        # Automatic sampling only when dataset exceeds thresholds
+        should_sample = file_size_mb > max_size_threshold or rows > max_rows_threshold
+        if should_sample and not minimal:
+            sample_size = int(os.getenv("YDATA_SAMPLE_SIZE", "100000"))
+            df = df.sample(n=min(sample_size, rows), random_state=42)
+            minimal = True  # Force minimal mode for large files
+        
+        # Force minimal mode for very large files even after sampling
+        if file_size_mb > max_size_threshold * 2:
+            minimal = True
+
+        if quiet:
+            # ydata-profiling can emit very verbose tqdm/progress output; keep CLI output clean by default.
+            os.environ.setdefault("TQDM_DISABLE", "1")
+            os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
+        
+        # Create output directory if needed
+        os.makedirs(os.path.dirname(output_path) or "./outputs/reports", exist_ok=True)
+        
+        # Configure profile based on minimal flag
+        output_capture = io.StringIO() if quiet else None
+        error_capture = io.StringIO() if quiet else None
+        if quiet:
+            output_ctx = contextlib.redirect_stdout(output_capture)
+            error_ctx = contextlib.redirect_stderr(error_capture)
+        else:
+            output_ctx = contextlib.nullcontext()
+            error_ctx = contextlib.nullcontext()
+
+        with output_ctx, error_ctx:
+            if minimal:
+                # Minimal mode: faster for large datasets, less memory
+                profile = ProfileReport(
+                    df,
+                    title=title,
+                    minimal=True,
+                    explorative=False,
+                    samples=None,  # Disable sample display to save memory
+                    correlations=None,  # Skip correlations in minimal mode
+                    missing_diagrams=None,  # Skip missing diagrams
+                    duplicates=None,  # Skip duplicate analysis
+                    interactions=None  # Skip interactions
+                )
+            else:
+                # Full mode: comprehensive analysis
+                profile = ProfileReport(
+                    df,
+                    title=title,
+                    explorative=True,
+                    correlations={
+                        "pearson": {"calculate": True},
+                        "spearman": {"calculate": True},
+                        "kendall": {"calculate": False},  # Slow for large datasets
+                        "phi_k": {"calculate": True},
+                        "cramers": {"calculate": True},
+                    }
+                )
+
+            # Generate HTML report
+            profile.to_file(output_path)
+        
+        # Extract key statistics
+        num_features = len(df.columns)
+        num_rows = len(df)
+        num_numeric = df.select_dtypes(include=['number']).shape[1]
+        num_categorical = df.select_dtypes(include=['object', 'category']).shape[1]
+        num_boolean = df.select_dtypes(include=['bool']).shape[1]
+        missing_cells = df.isnull().sum().sum()
+        total_cells = num_rows * num_features
+        missing_pct = (missing_cells / total_cells) * 100 if total_cells > 0 else 0
+        duplicate_rows = df.duplicated().sum()
+        
+        return {
+            "success": True,
+            "report_path": output_path,
+            "message": f"✅ ydata-profiling report generated successfully at: {output_path}",
+            "statistics": {
+                "dataset_size": {
+                    "rows": num_rows,
+                    "columns": num_features,
+                    "cells": total_cells
+                },
+                "variable_types": {
+                    "numeric": num_numeric,
+                    "categorical": num_categorical,
+                    "boolean": num_boolean
+                },
+                "data_quality": {
+                    "missing_cells": missing_cells,
+                    "missing_percentage": round(missing_pct, 2),
+                    "duplicate_rows": int(duplicate_rows)
+                },
+                "report_config": {
+                    "minimal_mode": minimal,
+                    "title": title
+                }
+            }
+        }
+        
+    except ImportError:
+        return {
+            "success": False,
+            "error": "ydata-profiling not installed. Install with: pip install ydata-profiling",
+            "error_type": "MissingDependency"
+        }
+    except Exception as e:
+        return {
+            "success": False,
+            "error": f"Failed to generate ydata-profiling report: {str(e)}",
+            "error_type": type(e).__name__
+        }
+
+
+def generate_sweetviz_report(
+    file_path: str,
+    target_col: Optional[str] = None,
+    compare_file_path: Optional[str] = None,
+    output_path: str = "./outputs/reports/sweetviz_report.html",
+    title: str = "Sweetviz EDA Report",
+    quiet: bool = True
+) -> Dict[str, Any]:
+    """
+    Generate an interactive EDA report using Sweetviz.
+    
+    Sweetviz provides:
+    - Feature-by-feature analysis with distributions
+    - Target analysis (associations with target variable)
+    - Dataset comparison (train vs test)
+    - Correlations/associations for numeric and categorical features
+    
+    Args:
+        file_path: Path to the dataset CSV file
+        target_col: Optional target column for supervised analysis
+        compare_file_path: Optional second dataset for comparison (e.g., test set)
+        output_path: Where to save the HTML report
+        title: Title for the report
+        
+    Returns:
+        Dict with success status and report path
+    """
+    try:
+        import sweetviz as sv
+        import pandas as pd
+    except ImportError:
+        return {
+            "success": False,
+            "error": "sweetviz not installed. Install with: pip install sweetviz>=2.3",
+            "error_type": "MissingDependency"
+        }
+    
+    try:
+        # Read dataset
+        if file_path.endswith('.csv'):
+            df = pd.read_csv(file_path)
+        elif file_path.endswith('.parquet'):
+            df = pd.read_parquet(file_path)
+        else:
+            raise ValueError(f"Unsupported file format: {file_path}")
+        
+        # Create output directory
+        os.makedirs(os.path.dirname(output_path) or "./outputs/reports", exist_ok=True)
+        
+        if quiet:
+            os.environ.setdefault("TQDM_DISABLE", "1")
+
+        output_capture = io.StringIO() if quiet else None
+        error_capture = io.StringIO() if quiet else None
+        if quiet:
+            output_ctx = contextlib.redirect_stdout(output_capture)
+            error_ctx = contextlib.redirect_stderr(error_capture)
+        else:
+            output_ctx = contextlib.nullcontext()
+            error_ctx = contextlib.nullcontext()
+
+        with output_ctx, error_ctx:
+            # Generate report
+            if compare_file_path:
+                # Comparison report (train vs test)
+                if compare_file_path.endswith('.csv'):
+                    df_compare = pd.read_csv(compare_file_path)
+                else:
+                    df_compare = pd.read_parquet(compare_file_path)
+
+                if target_col and target_col in df.columns:
+                    report = sv.compare([df, "Dataset 1"], [df_compare, "Dataset 2"], target_feat=target_col)
+                else:
+                    report = sv.compare([df, "Dataset 1"], [df_compare, "Dataset 2"])
+            else:
+                # Single dataset analysis
+                if target_col and target_col in df.columns:
+                    report = sv.analyze(df, target_feat=target_col)
+                else:
+                    report = sv.analyze(df)
+
+            # Save report (show_html=False prevents auto-opening browser)
+            report.show_html(output_path, open_browser=False)
+        
+        num_features = len(df.columns)
+        num_rows = len(df)
+        
+        return {
+            "success": True,
+            "report_path": output_path,
+            "message": f"✅ Sweetviz report generated at: {output_path}",
+            "statistics": {
+                "rows": num_rows,
+                "columns": num_features,
+                "target_column": target_col,
+                "comparison_mode": compare_file_path is not None
+            }
+        }
+        
+    except Exception as e:
+        return {
+            "success": False,
+            "error": f"Failed to generate Sweetviz report: {str(e)}",
+            "error_type": type(e).__name__
+        }

package/src/tools/enhanced_feature_engineering.py

@@ -0,0 +1,241 @@
+"""
+Enhanced Feature Engineering - Additional robust features
+"""
+
+import polars as pl
+import numpy as np
+from typing import Dict, Any, List, Optional
+from pathlib import Path
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from ds_agent.utils.polars_helpers import load_dataframe, save_dataframe, get_numeric_columns
+from ds_agent.utils.validation import validate_file_exists, validate_dataframe
+
+
+def create_ratio_features(file_path: str, 
+                          columns: Optional[List[str]] = None,
+                          max_ratios: int = 20,
+                          output_path: Optional[str] = None) -> Dict[str, Any]:
+    """
+    Create ratio features (a/b) for all numeric column pairs.
+    ROBUST: Handles division by zero, infinity, and NaN values.
+    
+    Args:
+        file_path: Path to dataset
+        columns: Columns to use (None = all numeric)
+        max_ratios: Maximum number of ratio features
+        output_path: Output file path
+        
+    Returns:
+        Dictionary with results
+    """
+    validate_file_exists(file_path)
+    df = load_dataframe(file_path)
+    validate_dataframe(df)
+    
+    if columns is None:
+        columns = get_numeric_columns(df)
+    
+    print(f"🔢 Creating ratio features from {len(columns)} columns...")
+    
+    ratio_exprs = []
+    feature_names = []
+    
+    for i, col1 in enumerate(columns[:15]):
+        for col2 in columns[i+1:16]:
+            if len(ratio_exprs) >= max_ratios:
+                break
+            
+            # Safe division (avoid div by zero, replace inf/nan)
+            ratio_name = f"ratio_{col1}_div_{col2}"
+            ratio_expr = (
+                pl.when(pl.col(col2).abs() < 1e-10)
+                .then(0)
+                .otherwise(pl.col(col1) / pl.col(col2))
+                .clip(-1e6, 1e6)  # Clip extreme values
+                .fill_nan(0)
+                .fill_null(0)
+                .alias(ratio_name)
+            )
+            ratio_exprs.append(ratio_expr)
+            feature_names.append(ratio_name)
+    
+    df = df.with_columns(ratio_exprs)
+    
+    if output_path:
+        save_dataframe(df, output_path)
+    
+    return {
+        'success': True,
+        'tool': 'create_ratio_features',
+        'result': {
+            'new_features': len(ratio_exprs),
+            'feature_names': feature_names,
+            'output_path': output_path
+        }
+    }
+
+
+def create_statistical_features(file_path: str,
+                                columns: Optional[List[str]] = None,
+                                output_path: Optional[str] = None) -> Dict[str, Any]:
+    """
+    Create row-wise statistical features (mean, std, min, max, range).
+    ROBUST: Handles missing values and edge cases.
+    
+    Args:
+        file_path: Path to dataset
+        columns: Columns to use (None = all numeric)
+        output_path: Output file path
+        
+    Returns:
+        Dictionary with results
+    """
+    validate_file_exists(file_path)
+    df = load_dataframe(file_path)
+    validate_dataframe(df)
+    
+    if columns is None:
+        columns = get_numeric_columns(df)
+    
+    print(f"📊 Creating statistical features across {len(columns)} columns...")
+    
+    # Row-wise statistics
+    stat_features = [
+        pl.concat_list([pl.col(c) for c in columns]).list.mean().fill_null(0).alias('row_mean'),
+        pl.concat_list([pl.col(c) for c in columns]).list.std().fill_null(0).alias('row_std'),
+        pl.concat_list([pl.col(c) for c in columns]).list.min().fill_null(0).alias('row_min'),
+        pl.concat_list([pl.col(c) for c in columns]).list.max().fill_null(0).alias('row_max'),
+        (pl.concat_list([pl.col(c) for c in columns]).list.max() - 
+         pl.concat_list([pl.col(c) for c in columns]).list.min()).fill_null(0).alias('row_range'),
+        pl.concat_list([pl.col(c) for c in columns]).list.sum().fill_null(0).alias('row_sum'),
+    ]
+    
+    df = df.with_columns(stat_features)
+    
+    if output_path:
+        save_dataframe(df, output_path)
+    
+    return {
+        'success': True,
+        'tool': 'create_statistical_features',
+        'result': {
+            'new_features': 6,
+            'feature_names': ['row_mean', 'row_std', 'row_min', 'row_max', 'row_range', 'row_sum'],
+            'output_path': output_path
+        }
+    }
+
+
+def create_log_features(file_path: str,
+                       columns: Optional[List[str]] = None,
+                       output_path: Optional[str] = None) -> Dict[str, Any]:
+    """
+    Create log-transformed features for skewed distributions.
+    ROBUST: Handles negative values and zeros.
+    
+    Args:
+        file_path: Path to dataset
+        columns: Columns to use (None = all numeric with positive values)
+        output_path: Output file path
+        
+    Returns:
+        Dictionary with results
+    """
+    validate_file_exists(file_path)
+    df = load_dataframe(file_path)
+    validate_dataframe(df)
+    
+    if columns is None:
+        columns = get_numeric_columns(df)
+    
+    print(f"📈 Creating log-transformed features for {len(columns)} columns...")
+    
+    log_exprs = []
+    feature_names = []
+    
+    for col in columns:
+        # Check if column has positive values
+        min_val = df[col].min()
+        if min_val is not None and min_val > 0:
+            # log(x)
+            log_exprs.append(pl.col(col).log().fill_nan(0).alias(f"log_{col}"))
+            feature_names.append(f"log_{col}")
+        elif min_val is not None and min_val >= 0:
+            # log(x+1) for non-negative values
+            log_exprs.append((pl.col(col) + 1).log().fill_nan(0).alias(f"log1p_{col}"))
+            feature_names.append(f"log1p_{col}")
+    
+    if log_exprs:
+        df = df.with_columns(log_exprs)
+    
+    if output_path:
+        save_dataframe(df, output_path)
+    
+    return {
+        'success': True,
+        'tool': 'create_log_features',
+        'result': {
+            'new_features': len(log_exprs),
+            'feature_names': feature_names,
+            'output_path': output_path
+        }
+    }
+
+
+def create_binned_features(file_path: str,
+                          columns: Optional[List[str]] = None,
+                          n_bins: int = 5,
+                          output_path: Optional[str] = None) -> Dict[str, Any]:
+    """
+    Create binned (discretized) features from continuous variables.
+    ROBUST: Uses quantile-based binning to handle outliers.
+    
+    Args:
+        file_path: Path to dataset
+        columns: Columns to use (None = all numeric)
+        n_bins: Number of bins
+        output_path: Output file path
+        
+    Returns:
+        Dictionary with results
+    """
+    validate_file_exists(file_path)
+    df = load_dataframe(file_path)
+    validate_dataframe(df)
+    
+    if columns is None:
+        columns = get_numeric_columns(df)[:10]  # Limit to 10 columns
+    
+    print(f"🗂️  Creating binned features for {len(columns)} columns with {n_bins} bins...")
+    
+    binned_exprs = []
+    feature_names = []
+    
+    for col in columns:
+        # Quantile-based binning
+        bin_name = f"{col}_binned"
+        binned_exprs.append(
+            pl.col(col).qcut(n_bins, labels=[f"bin_{i}" for i in range(n_bins)], 
+                            allow_duplicates=True).fill_null("bin_0").alias(bin_name)
+        )
+        feature_names.append(bin_name)
+    
+    df = df.with_columns(binned_exprs)
+    
+    if output_path:
+        save_dataframe(df, output_path)
+    
+    return {
+        'success': True,
+        'tool': 'create_binned_features',
+        'result': {
+            'new_features': len(binned_exprs),
+            'feature_names': feature_names,
+            'n_bins': n_bins,
+            'output_path': output_path
+        }
+    }