claude-flow-novice 1.2.1 → 1.3.0

package/examples/refinement_agent_workdir/ablation_framework.py

@@ -0,0 +1,333 @@
+#!/usr/bin/env python3
+"""
+MLE-STAR Ablation Analysis Framework
+Refinement Agent Component Impact Analysis
+"""
+
+import numpy as np
+import pandas as pd
+from sklearn.base import BaseEstimator, TransformerMixin
+from sklearn.model_selection import cross_val_score, KFold
+from sklearn.metrics import accuracy_score, mean_squared_error, r2_score
+import json
+import time
+from typing import Dict, List, Any, Tuple, Optional
+import logging
+from dataclasses import dataclass, asdict
+from datetime import datetime
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+logger = logging.getLogger('ablation_framework')
+
+
+@dataclass
+class ComponentResult:
+    """Results from testing a single component configuration"""
+    component_name: str
+    configuration: Dict[str, Any]
+    performance_metrics: Dict[str, float]
+    impact_score: float
+    computational_time: float
+    timestamp: str
+
+
+class AblationAnalyzer:
+    """
+    Systematic ablation analysis for ML pipeline components
+    Identifies which components have the highest impact on performance
+    """
+    
+    def __init__(self, baseline_pipeline: Dict[str, Any], 
+                 evaluation_metric: str = 'accuracy',
+                 cv_folds: int = 5,
+                 random_state: int = 42):
+        """
+        Initialize ablation analyzer
+        
+        Args:
+            baseline_pipeline: Dictionary containing pipeline components
+            evaluation_metric: Metric to optimize ('accuracy', 'mse', 'r2')
+            cv_folds: Number of cross-validation folds
+            random_state: Random seed for reproducibility
+        """
+        self.baseline_pipeline = baseline_pipeline
+        self.evaluation_metric = evaluation_metric
+        self.cv_folds = cv_folds
+        self.random_state = random_state
+        self.results = []
+        self.baseline_performance = None
+        
+    def evaluate_pipeline(self, pipeline: Dict[str, Any], 
+                         X: np.ndarray, y: np.ndarray) -> Tuple[Dict[str, float], float]:
+        """
+        Evaluate a pipeline configuration
+        
+        Returns:
+            Tuple of (metrics_dict, computation_time)
+        """
+        start_time = time.time()
+        
+        # Create cross-validation splitter
+        cv = KFold(n_splits=self.cv_folds, shuffle=True, random_state=self.random_state)
+        
+        # Get the model from pipeline
+        model = pipeline.get('model')
+        preprocessor = pipeline.get('preprocessor')
+        feature_engineer = pipeline.get('feature_engineer')
+        
+        # Apply preprocessing if available
+        X_processed = X.copy()
+        if preprocessor:
+            X_processed = preprocessor.fit_transform(X_processed)
+            
+        # Apply feature engineering if available
+        if feature_engineer:
+            X_processed = feature_engineer.fit_transform(X_processed)
+        
+        # Evaluate model
+        if self.evaluation_metric == 'accuracy':
+            scores = cross_val_score(model, X_processed, y, cv=cv, scoring='accuracy')
+            metric_value = scores.mean()
+            metric_std = scores.std()
+        elif self.evaluation_metric == 'mse':
+            scores = cross_val_score(model, X_processed, y, cv=cv, 
+                                   scoring='neg_mean_squared_error')
+            metric_value = -scores.mean()  # Convert back to positive
+            metric_std = scores.std()
+        elif self.evaluation_metric == 'r2':
+            scores = cross_val_score(model, X_processed, y, cv=cv, scoring='r2')
+            metric_value = scores.mean()
+            metric_std = scores.std()
+        
+        computation_time = time.time() - start_time
+        
+        metrics = {
+            f'{self.evaluation_metric}_mean': metric_value,
+            f'{self.evaluation_metric}_std': metric_std,
+            'cv_scores': scores.tolist()
+        }
+        
+        return metrics, computation_time
+    
+    def ablate_component(self, component_name: str, 
+                        ablation_configs: List[Dict[str, Any]],
+                        X: np.ndarray, y: np.ndarray) -> List[ComponentResult]:
+        """
+        Test different configurations for a single component
+        
+        Args:
+            component_name: Name of component to ablate
+            ablation_configs: List of configurations to test
+            X: Feature matrix
+            y: Target vector
+            
+        Returns:
+            List of ComponentResult objects
+        """
+        component_results = []
+        
+        logger.info(f"Starting ablation analysis for component: {component_name}")
+        
+        for config in ablation_configs:
+            # Create modified pipeline
+            modified_pipeline = self.baseline_pipeline.copy()
+            
+            # Apply configuration
+            if config is None:
+                # Remove component entirely
+                modified_pipeline[component_name] = None
+                config_desc = "removed"
+            else:
+                # Update component configuration
+                modified_pipeline[component_name] = config
+                config_desc = str(config)
+            
+            # Evaluate modified pipeline
+            try:
+                metrics, comp_time = self.evaluate_pipeline(modified_pipeline, X, y)
+                
+                # Calculate impact score
+                if self.baseline_performance is None:
+                    impact_score = 0.0
+                else:
+                    baseline_value = self.baseline_performance[f'{self.evaluation_metric}_mean']
+                    current_value = metrics[f'{self.evaluation_metric}_mean']
+                    
+                    # Calculate relative impact
+                    if self.evaluation_metric in ['accuracy', 'r2']:
+                        # Higher is better
+                        impact_score = (current_value - baseline_value) / baseline_value
+                    else:
+                        # Lower is better (MSE)
+                        impact_score = (baseline_value - current_value) / baseline_value
+                
+                result = ComponentResult(
+                    component_name=component_name,
+                    configuration={'config': config_desc},
+                    performance_metrics=metrics,
+                    impact_score=impact_score,
+                    computational_time=comp_time,
+                    timestamp=datetime.now().isoformat()
+                )
+                
+                component_results.append(result)
+                self.results.append(result)
+                
+                logger.info(f"  Config: {config_desc} | Impact: {impact_score:.4f} | "
+                          f"Performance: {metrics[f'{self.evaluation_metric}_mean']:.4f}")
+                
+            except Exception as e:
+                logger.error(f"Error evaluating config {config_desc}: {str(e)}")
+                
+        return component_results
+    
+    def run_full_ablation(self, components_to_test: Dict[str, List[Dict[str, Any]]],
+                         X: np.ndarray, y: np.ndarray) -> Dict[str, Any]:
+        """
+        Run ablation analysis on all specified components
+        
+        Args:
+            components_to_test: Dict mapping component names to configurations
+            X: Feature matrix
+            y: Target vector
+            
+        Returns:
+            Analysis results dictionary
+        """
+        logger.info("Starting full ablation analysis")
+        
+        # First, establish baseline performance
+        logger.info("Establishing baseline performance...")
+        self.baseline_performance, baseline_time = self.evaluate_pipeline(
+            self.baseline_pipeline, X, y
+        )
+        logger.info(f"Baseline {self.evaluation_metric}: "
+                   f"{self.baseline_performance[f'{self.evaluation_metric}_mean']:.4f}")
+        
+        # Test each component
+        component_impacts = {}
+        for component_name, configs in components_to_test.items():
+            results = self.ablate_component(component_name, configs, X, y)
+            
+            # Calculate average impact for this component
+            if results:
+                avg_impact = np.mean([r.impact_score for r in results])
+                max_impact = max([r.impact_score for r in results])
+                component_impacts[component_name] = {
+                    'average_impact': avg_impact,
+                    'max_impact': max_impact,
+                    'num_configs_tested': len(results)
+                }
+        
+        # Rank components by impact
+        ranked_components = sorted(
+            component_impacts.items(),
+            key=lambda x: abs(x[1]['max_impact']),
+            reverse=True
+        )
+        
+        # Prepare analysis summary
+        analysis_summary = {
+            'baseline_performance': self.baseline_performance,
+            'components_tested': list(components_to_test.keys()),
+            'total_configurations': len(self.results),
+            'component_impacts': component_impacts,
+            'component_rankings': [
+                {
+                    'rank': i + 1,
+                    'component': comp[0],
+                    'max_impact': comp[1]['max_impact'],
+                    'average_impact': comp[1]['average_impact']
+                }
+                for i, comp in enumerate(ranked_components)
+            ],
+            'highest_impact_component': ranked_components[0][0] if ranked_components else None,
+            'improvement_opportunities': self._identify_improvements(),
+            'timestamp': datetime.now().isoformat()
+        }
+        
+        return analysis_summary
+    
+    def _identify_improvements(self) -> List[Dict[str, Any]]:
+        """Identify specific improvement opportunities from results"""
+        improvements = []
+        
+        # Find configurations that improved performance
+        for result in self.results:
+            if result.impact_score > 0.01:  # At least 1% improvement
+                improvements.append({
+                    'component': result.component_name,
+                    'configuration': result.configuration,
+                    'impact': result.impact_score,
+                    'performance': result.performance_metrics[f'{self.evaluation_metric}_mean']
+                })
+        
+        # Sort by impact
+        improvements.sort(key=lambda x: x['impact'], reverse=True)
+        
+        return improvements[:10]  # Top 10 improvements
+    
+    def export_results(self, filepath: str):
+        """Export ablation analysis results to JSON file"""
+        export_data = {
+            'baseline_performance': self.baseline_performance,
+            'results': [asdict(r) for r in self.results],
+            'summary': {
+                'total_tests': len(self.results),
+                'evaluation_metric': self.evaluation_metric,
+                'cv_folds': self.cv_folds,
+                'timestamp': datetime.now().isoformat()
+            }
+        }
+        
+        with open(filepath, 'w') as f:
+            json.dump(export_data, f, indent=2)
+        
+        logger.info(f"Results exported to {filepath}")
+
+
+class FeatureEngineer(BaseEstimator, TransformerMixin):
+    """Custom feature engineering transformer for ablation testing"""
+    
+    def __init__(self, polynomial_features: bool = False,
+                 interaction_features: bool = False,
+                 log_transform: bool = False):
+        self.polynomial_features = polynomial_features
+        self.interaction_features = interaction_features
+        self.log_transform = log_transform
+        
+    def fit(self, X, y=None):
+        return self
+        
+    def transform(self, X):
+        X_transformed = X.copy()
+        
+        if self.polynomial_features:
+            # Add polynomial features (squared terms)
+            poly_features = X_transformed ** 2
+            X_transformed = np.hstack([X_transformed, poly_features])
+            
+        if self.interaction_features and X.shape[1] > 1:
+            # Add interaction features (pairwise products)
+            interactions = []
+            for i in range(X.shape[1]):
+                for j in range(i + 1, X.shape[1]):
+                    interactions.append(X[:, i] * X[:, j])
+            if interactions:
+                X_transformed = np.hstack([X_transformed, np.array(interactions).T])
+                
+        if self.log_transform:
+            # Add log-transformed features (for positive values)
+            positive_mask = X > 0
+            log_features = np.zeros_like(X)
+            log_features[positive_mask] = np.log(X[positive_mask])
+            X_transformed = np.hstack([X_transformed, log_features])
+            
+        return X_transformed
+
+
+if __name__ == "__main__":
+    # Example usage
+    logger.info("MLE-STAR Ablation Framework initialized")
+    logger.info("Ready for component impact analysis")

package/examples/refinement_agent_workdir/refinement_demo.py

@@ -0,0 +1,345 @@
+#!/usr/bin/env python3
+"""
+MLE-STAR Refinement Agent Demonstration
+Shows how to use ablation analysis and targeted optimization
+"""
+
+import numpy as np
+import pandas as pd
+from sklearn.datasets import make_classification, load_iris
+from sklearn.model_selection import train_test_split
+from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler
+from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
+from sklearn.linear_model import LogisticRegression
+from sklearn.svm import SVC
+import json
+import subprocess
+import os
+from datetime import datetime
+
+# Import our custom components
+from ablation_framework import AblationAnalyzer, FeatureEngineer
+from targeted_optimizer import TargetedOptimizer, AdaptiveOptimizer
+
+# Configure logging
+import logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger('refinement_demo')
+
+
+def store_to_memory(key: str, value: dict):
+    """Store results to Claude Flow memory"""
+    try:
+        value_str = json.dumps(value)
+        cmd = f'npx claude-flow@alpha memory store "agent/refinement_agent/{key}" \'{value_str}\''
+        subprocess.run(cmd, shell=True, capture_output=True)
+        logger.info(f"Stored {key} to memory")
+    except Exception as e:
+        logger.error(f"Failed to store to memory: {e}")
+
+
+def run_refinement_workflow():
+    """
+    Complete refinement workflow demonstrating:
+    1. Ablation analysis to identify high-impact components
+    2. Targeted optimization of those components
+    3. Iterative refinement process
+    """
+    
+    logger.info("=== MLE-STAR Refinement Agent Demo ===")
+    logger.info("Starting refinement workflow...")
+    
+    # Step 1: Create synthetic dataset for demonstration
+    logger.info("\n--- Step 1: Dataset Preparation ---")
+    X, y = make_classification(
+        n_samples=1000,
+        n_features=20,
+        n_informative=15,
+        n_redundant=5,
+        n_classes=3,
+        random_state=42
+    )
+    
+    X_train, X_test, y_train, y_test = train_test_split(
+        X, y, test_size=0.2, random_state=42, stratify=y
+    )
+    
+    logger.info(f"Dataset shape: {X_train.shape}")
+    logger.info(f"Classes: {np.unique(y_train)}")
+    
+    # Step 2: Define baseline pipeline
+    logger.info("\n--- Step 2: Baseline Pipeline Setup ---")
+    baseline_pipeline = {
+        'preprocessor': StandardScaler(),
+        'feature_engineer': FeatureEngineer(
+            polynomial_features=False,
+            interaction_features=False,
+            log_transform=False
+        ),
+        'model': RandomForestClassifier(n_estimators=100, random_state=42)
+    }
+    
+    # Step 3: Run ablation analysis
+    logger.info("\n--- Step 3: Ablation Analysis ---")
+    ablation_analyzer = AblationAnalyzer(
+        baseline_pipeline=baseline_pipeline,
+        evaluation_metric='accuracy',
+        cv_folds=5
+    )
+    
+    # Define component variations to test
+    components_to_test = {
+        'preprocessor': [
+            None,  # No preprocessing
+            StandardScaler(),
+            MinMaxScaler(),
+            RobustScaler()
+        ],
+        'feature_engineer': [
+            None,  # No feature engineering
+            FeatureEngineer(polynomial_features=True),
+            FeatureEngineer(interaction_features=True),
+            FeatureEngineer(log_transform=True),
+            FeatureEngineer(polynomial_features=True, interaction_features=True)
+        ],
+        'model': [
+            LogisticRegression(random_state=42),
+            RandomForestClassifier(n_estimators=50, random_state=42),
+            RandomForestClassifier(n_estimators=200, random_state=42),
+            GradientBoostingClassifier(n_estimators=100, random_state=42),
+            SVC(kernel='rbf', random_state=42)
+        ]
+    }
+    
+    # Run ablation analysis
+    ablation_results = ablation_analyzer.run_full_ablation(
+        components_to_test=components_to_test,
+        X=X_train,
+        y=y_train
+    )
+    
+    # Save ablation results
+    ablation_analyzer.export_results('ablation_results.json')
+    store_to_memory('ablation_results', ablation_results)
+    
+    logger.info("\n--- Ablation Analysis Results ---")
+    logger.info(f"Baseline performance: {ablation_results['baseline_performance']['accuracy_mean']:.4f}")
+    logger.info("\nComponent Rankings by Impact:")
+    for ranking in ablation_results['component_rankings']:
+        logger.info(f"  {ranking['rank']}. {ranking['component']}: "
+                   f"max_impact={ranking['max_impact']:.4f}, "
+                   f"avg_impact={ranking['average_impact']:.4f}")
+    
+    # Step 4: Targeted optimization on highest impact component
+    logger.info("\n--- Step 4: Targeted Optimization ---")
+    highest_impact_component = ablation_results['highest_impact_component']
+    logger.info(f"Focusing optimization on: {highest_impact_component}")
+    
+    # Initialize targeted optimizer
+    targeted_optimizer = TargetedOptimizer(
+        baseline_score=ablation_results['baseline_performance']['accuracy_mean'],
+        evaluation_metric='accuracy',
+        cv_folds=5
+    )
+    
+    # Define optimization configurations based on highest impact component
+    if highest_impact_component == 'model':
+        # Optimize model hyperparameters
+        logger.info("Optimizing model hyperparameters...")
+        
+        # Random Forest optimization
+        rf_params = {
+            'n_estimators': [100, 200, 300],
+            'max_depth': [10, 20, 30, None],
+            'min_samples_split': [2, 5, 10],
+            'min_samples_leaf': [1, 2, 4],
+            'max_features': ['sqrt', 'log2', None]
+        }
+        
+        rf_result = targeted_optimizer.optimize_hyperparameters_grid(
+            estimator=RandomForestClassifier(random_state=42),
+            param_grid=rf_params,
+            X=X_train,
+            y=y_train,
+            component_name='random_forest_model'
+        )
+        
+        # Gradient Boosting optimization with Bayesian search
+        from skopt.space import Real, Integer
+        gb_search_spaces = {
+            'n_estimators': Integer(50, 300),
+            'learning_rate': Real(0.01, 0.3, prior='log-uniform'),
+            'max_depth': Integer(3, 10),
+            'min_samples_split': Integer(2, 20),
+            'subsample': Real(0.6, 1.0)
+        }
+        
+        gb_result = targeted_optimizer.optimize_hyperparameters_bayesian(
+            estimator=GradientBoostingClassifier(random_state=42),
+            search_spaces=gb_search_spaces,
+            X=X_train,
+            y=y_train,
+            component_name='gradient_boosting_model',
+            n_iter=30
+        )
+        
+    elif highest_impact_component == 'feature_engineer':
+        # Optimize feature engineering
+        logger.info("Optimizing feature engineering strategies...")
+        
+        fe_result = targeted_optimizer.optimize_feature_engineering(
+            X=X_train,
+            y=y_train,
+            base_model=RandomForestClassifier(n_estimators=100, random_state=42)
+        )
+        
+    # Step 5: Iterative refinement
+    logger.info("\n--- Step 5: Iterative Refinement ---")
+    
+    # Set up components for iterative refinement
+    refinement_components = {
+        'random_forest': {
+            'estimator': RandomForestClassifier(random_state=42),
+            'method': 'grid',
+            'param_grid': {
+                'n_estimators': [150, 200, 250],
+                'max_depth': [15, 20, 25],
+                'min_samples_split': [2, 3, 4]
+            }
+        },
+        'gradient_boosting': {
+            'estimator': GradientBoostingClassifier(random_state=42),
+            'method': 'bayesian',
+            'search_spaces': {
+                'n_estimators': Integer(100, 200),
+                'learning_rate': Real(0.05, 0.2),
+                'max_depth': Integer(4, 8)
+            },
+            'n_iter': 20
+        }
+    }
+    
+    iterative_results = targeted_optimizer.iterative_refinement(
+        components=refinement_components,
+        X=X_train,
+        y=y_train,
+        max_iterations=3
+    )
+    
+    # Save optimization results
+    targeted_optimizer.export_results('optimization_results.json')
+    store_to_memory('optimization_results', iterative_results)
+    
+    logger.info("\n--- Optimization Results Summary ---")
+    logger.info(f"Baseline score: {targeted_optimizer.baseline_score:.4f}")
+    logger.info(f"Final best score: {iterative_results['final_best_score']:.4f}")
+    logger.info(f"Total improvement: {iterative_results['total_improvement']:.2%}")
+    
+    # Step 6: Create final optimized pipeline
+    logger.info("\n--- Step 6: Final Optimized Pipeline ---")
+    
+    best_configs = targeted_optimizer._extract_best_configs()
+    logger.info("\nBest configurations found:")
+    for component, config in best_configs.items():
+        logger.info(f"  {component}: score={config['score']:.4f}, method={config['method']}")
+        logger.info(f"    params: {config['params']}")
+    
+    # Store final results
+    final_summary = {
+        'workflow': 'MLE-STAR_refinement',
+        'session_id': 'automation-session-1754319839721-scewi2uw3',
+        'agent': 'refinement_agent',
+        'baseline_score': targeted_optimizer.baseline_score,
+        'final_score': iterative_results['final_best_score'],
+        'improvement': iterative_results['total_improvement'],
+        'highest_impact_component': highest_impact_component,
+        'optimization_time': sum(r.optimization_time for r in targeted_optimizer.optimization_history),
+        'timestamp': datetime.now().isoformat()
+    }
+    
+    store_to_memory('workflow_summary', final_summary)
+    
+    logger.info("\n=== Refinement Workflow Complete ===")
+    logger.info(f"Total improvement achieved: {iterative_results['total_improvement']:.2%}")
+    
+    return final_summary
+
+
+def demonstrate_adaptive_optimization():
+    """
+    Demonstrate adaptive optimization that automatically
+    chooses the best optimization method
+    """
+    logger.info("\n=== Adaptive Optimization Demo ===")
+    
+    # Create dataset
+    X, y = load_iris(return_X_y=True)
+    X_train, X_test, y_train, y_test = train_test_split(
+        X, y, test_size=0.2, random_state=42
+    )
+    
+    # Initialize optimizers
+    targeted_opt = TargetedOptimizer(
+        baseline_score=0.9,  # Hypothetical baseline
+        evaluation_metric='accuracy'
+    )
+    adaptive_opt = AdaptiveOptimizer(targeted_opt)
+    
+    # Define different search spaces
+    search_spaces = {
+        'small_discrete': {
+            'n_estimators': [50, 100, 150],
+            'max_depth': [5, 10, 15]
+        },
+        'large_mixed': {
+            'n_estimators': (50, 300),
+            'learning_rate': (0.01, 0.3),
+            'max_depth': [3, 5, 7, 10, 15, 20],
+            'min_samples_split': (2, 20)
+        },
+        'continuous_only': {
+            'C': (0.01, 100.0),
+            'gamma': (0.0001, 1.0)
+        }
+    }
+    
+    # Test adaptive optimization on different search spaces
+    for space_name, space in search_spaces.items():
+        logger.info(f"\nOptimizing with {space_name} search space...")
+        
+        if space_name == 'continuous_only':
+            estimator = SVC(kernel='rbf', random_state=42)
+        else:
+            estimator = RandomForestClassifier(random_state=42)
+        
+        result = adaptive_opt.optimize_adaptive(
+            component_name=f'adaptive_{space_name}',
+            search_space=space,
+            estimator=estimator,
+            X=X_train,
+            y=y_train
+        )
+        
+        logger.info(f"  Method used: {result.optimization_method}")
+        logger.info(f"  Best score: {result.best_score:.4f}")
+        logger.info(f"  Improvement: {result.improvement_from_baseline:.2%}")
+
+
+if __name__ == "__main__":
+    # Run main refinement workflow
+    summary = run_refinement_workflow()
+    
+    # Demonstrate adaptive optimization
+    demonstrate_adaptive_optimization()
+    
+    # Final coordination hook
+    try:
+        subprocess.run(
+            'npx claude-flow@alpha hooks post-task --task-id "refinement_agent" '
+            '--analyze-performance true',
+            shell=True
+        )
+    except:
+        pass
+    
+    logger.info("\nRefinement agent work complete!")

package/examples/refinement_agent_workdir/requirements.txt

@@ -0,0 +1,9 @@
+# MLE-STAR Refinement Agent Requirements
+numpy>=1.21.0
+pandas>=1.3.0
+scikit-learn>=1.0.0
+scikit-optimize>=0.9.0
+optuna>=3.0.0
+matplotlib>=3.4.0
+seaborn>=0.11.0
+joblib>=1.1.0