pangea 0.0.45 → 0.0.47

data/lib/pangea/components/sustainable_ml_training/modules/code_generators/carbon_scheduler.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+# Copyright 2025 The Pangea Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Pangea
+  module Components
+    module SustainableMLTraining
+      # Carbon scheduler Lambda code generator
+      module CarbonSchedulerCode
+        def generate_carbon_scheduler_code(_input)
+          <<~PYTHON
+            import json
+            import boto3
+            import os
+            from datetime import datetime, timedelta
+
+            sagemaker = boto3.client('sagemaker')
+            dynamodb = boto3.resource('dynamodb')
+            cloudwatch = boto3.client('cloudwatch')
+
+            STATE_TABLE = os.environ['STATE_TABLE']
+            CARBON_TABLE = os.environ['CARBON_TABLE']
+            CARBON_THRESHOLD = int(os.environ['CARBON_THRESHOLD'])
+            PREFERRED_REGIONS = os.environ['PREFERRED_REGIONS'].split(',')
+            TRAINING_STRATEGY = os.environ['TRAINING_STRATEGY']
+
+            def handler(event, context):
+                state_table = dynamodb.Table(STATE_TABLE)
+                carbon_table = dynamodb.Table(CARBON_TABLE)
+                pending_jobs = get_pending_training_jobs(state_table)
+
+                for job in pending_jobs:
+                    carbon_data = get_regional_carbon_data(carbon_table)
+                    optimal_config = find_optimal_training_config(job, carbon_data, TRAINING_STRATEGY)
+
+                    if optimal_config['start_now']:
+                        start_training_job(job, optimal_config['region'])
+                        update_job_state(state_table, job['job_id'], 'training', optimal_config)
+                    else:
+                        schedule_training_job(job, optimal_config['start_time'], optimal_config['region'])
+
+                return {'statusCode': 200, 'body': json.dumps(f'Processed {len(pending_jobs)} jobs')}
+
+            def get_pending_training_jobs(table):
+                response = table.query(
+                    IndexName='status-index',
+                    KeyConditionExpression='#status = :status',
+                    ExpressionAttributeNames={'#status': 'status'},
+                    ExpressionAttributeValues={':status': 'pending'}
+                )
+                return response.get('Items', [])
+
+            def get_regional_carbon_data(table):
+                carbon_data = {}
+                current_time = int(datetime.now().timestamp())
+                for region in PREFERRED_REGIONS:
+                    response = table.query(
+                        KeyConditionExpression='region = :region AND #ts > :ts',
+                        ExpressionAttributeNames={'#ts': 'timestamp'},
+                        ExpressionAttributeValues={':region': region, ':ts': current_time - 900},
+                        ScanIndexForward=False, Limit=1
+                    )
+                    carbon_data[region] = response['Items'][0] if response['Items'] else estimate_carbon_intensity(region)
+                return carbon_data
+
+            def estimate_carbon_intensity(region):
+                baselines = {'us-west-2': 50, 'eu-north-1': 40, 'ca-central-1': 30, 'eu-west-1': 80, 'us-east-1': 400}
+                base = baselines.get(region, 300)
+                hour = datetime.now().hour
+                multiplier = 1.3 if 9 <= hour <= 17 else (0.7 if 0 <= hour <= 6 else 1.0)
+                return {'carbon_intensity': int(base * multiplier), 'renewable_percentage': 100 - (base / 5), 'timestamp': int(datetime.now().timestamp())}
+
+            def find_optimal_training_config(job, carbon_data, strategy):
+                strategies = {'carbon_aware_scheduling': carbon_aware_scheduling, 'efficient_architecture': efficient_architecture_scheduling,
+                              'mixed_precision': mixed_precision_scheduling, 'federated_learning': federated_learning_config}
+                return strategies.get(strategy, default_scheduling)(job, carbon_data)
+
+            def carbon_aware_scheduling(job, carbon_data):
+                best_region, min_carbon = min(carbon_data.items(), key=lambda x: x[1]['carbon_intensity'])
+                if min_carbon['carbon_intensity'] <= CARBON_THRESHOLD:
+                    return {'start_now': True, 'region': best_region, 'carbon_intensity': min_carbon['carbon_intensity'], 'strategy': 'immediate_low_carbon'}
+                return {'start_now': False, 'start_time': predict_low_carbon_window(best_region), 'region': best_region, 'carbon_intensity': min_carbon['carbon_intensity'], 'strategy': 'delayed_optimization'}
+
+            def efficient_architecture_scheduling(job, carbon_data):
+                for region in ['us-west-2', 'eu-north-1']:
+                    if region in carbon_data:
+                        return {'start_now': True, 'region': region, 'carbon_intensity': carbon_data[region]['carbon_intensity'], 'hardware_efficiency': 'optimized', 'strategy': 'efficient_hardware'}
+                return {'start_now': True, 'region': PREFERRED_REGIONS[0], 'carbon_intensity': 0, 'strategy': 'efficient_hardware'}
+
+            def mixed_precision_scheduling(job, carbon_data):
+                adjusted_threshold = CARBON_THRESHOLD * 1.4
+                for region, data in carbon_data.items():
+                    effective_carbon = data['carbon_intensity'] * 0.6
+                    if effective_carbon <= adjusted_threshold:
+                        return {'start_now': True, 'region': region, 'carbon_intensity': data['carbon_intensity'], 'effective_carbon': effective_carbon, 'optimization': 'mixed_precision', 'strategy': 'compute_optimized'}
+                return {'start_now': False, 'start_time': datetime.now() + timedelta(hours=2), 'region': PREFERRED_REGIONS[0], 'strategy': 'wait_for_cleaner_grid'}
+
+            def federated_learning_config(job, carbon_data):
+                eligible_regions = [r for r, d in carbon_data.items() if d['carbon_intensity'] < CARBON_THRESHOLD * 1.5]
+                if len(eligible_regions) >= 2:
+                    return {'start_now': True, 'regions': eligible_regions[:3], 'strategy': 'federated_distribution', 'carbon_intensity': sum(carbon_data[r]['carbon_intensity'] for r in eligible_regions[:3]) / 3}
+                return default_scheduling(job, carbon_data)
+
+            def default_scheduling(job, carbon_data):
+                best_region = min(carbon_data.items(), key=lambda x: x[1]['carbon_intensity'])[0]
+                return {'start_now': True, 'region': best_region, 'carbon_intensity': carbon_data[best_region]['carbon_intensity'], 'strategy': 'default'}
+
+            def predict_low_carbon_window(region):
+                current_hour = datetime.now().hour
+                hours_until_low = 24 - current_hour + 2 if current_hour >= 17 else max(20 - current_hour, 1)
+                return datetime.now() + timedelta(hours=hours_until_low)
+
+            def start_training_job(job, region):
+                job_config = job['training_config']
+                job_config['ResourceConfig']['InstanceType'] = select_regional_instance(region)
+                job_config['Environment'] = {'CARBON_REGION': region, 'CARBON_TRACKING': 'enabled', 'MODEL_OPTIMIZATION': job.get('optimization', 'mixed_precision')}
+                response = sagemaker.create_training_job(**job_config)
+                emit_scheduling_metrics(job, region)
+                return response['TrainingJobArn']
+
+            def select_regional_instance(region):
+                availability = {'us-west-2': ['ml.p4d.24xlarge'], 'eu-north-1': ['ml.p3.16xlarge'], 'ca-central-1': ['ml.g5.48xlarge'], 'eu-west-1': ['ml.p3.16xlarge'], 'us-east-1': ['ml.p4d.24xlarge']}
+                return availability.get(region, ['ml.p3.8xlarge'])[0]
+
+            def schedule_training_job(job, start_time, region):
+                events = boto3.client('events')
+                rule_name = f"training-{job['job_id']}-{int(start_time.timestamp())}"
+                events.put_rule(Name=rule_name, ScheduleExpression=f"at({start_time.strftime('%Y-%m-%dT%H:%M:%S')})", State='ENABLED')
+
+            def update_job_state(table, job_id, status, config):
+                table.put_item(Item={'job_id': job_id, 'timestamp': int(datetime.now().timestamp()), 'status': status, 'region': config.get('region', 'multi'), 'carbon_intensity': config.get('carbon_intensity', 0), 'optimization_strategy': config.get('strategy', 'default'), 'config': json.dumps(config)})
+
+            def emit_scheduling_metrics(job, region):
+                cloudwatch.put_metric_data(Namespace=f"SustainableML/{os.environ.get('COMPONENT_NAME', 'default')}", MetricData=[{'MetricName': 'TrainingJobsScheduled', 'Value': 1, 'Unit': 'Count', 'Dimensions': [{'Name': 'Region', 'Value': region}, {'Name': 'Strategy', 'Value': TRAINING_STRATEGY}]}])
+          PYTHON
+        end
+      end
+    end
+  end
+end

data/lib/pangea/components/sustainable_ml_training/modules/code_generators/efficiency_monitor.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+# Copyright 2025 The Pangea Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Pangea
+  module Components
+    module SustainableMLTraining
+      # Efficiency monitor Lambda code generator
+      module EfficiencyMonitorCode
+        def generate_efficiency_monitor_code(_input)
+          <<~PYTHON
+            import json
+            import boto3
+            import os
+            from datetime import datetime
+
+            cloudwatch = boto3.client('cloudwatch')
+            dynamodb = boto3.resource('dynamodb')
+
+            CARBON_TABLE = os.environ['CARBON_TABLE']
+            TRACK_CARBON = os.environ['TRACK_CARBON'] == 'True'
+            TRACK_ENERGY = os.environ['TRACK_ENERGY'] == 'True'
+            MODEL_TYPE = os.environ['MODEL_TYPE']
+
+            def handler(event, context):
+                carbon_table = dynamodb.Table(CARBON_TABLE)
+                metrics = collect_efficiency_metrics()
+                if TRACK_CARBON:
+                    carbon_metrics = calculate_carbon_emissions(metrics)
+                    metrics.update(carbon_metrics)
+                    store_carbon_data(carbon_table, carbon_metrics)
+                if TRACK_ENERGY:
+                    metrics.update(calculate_energy_usage(metrics))
+                metrics.update(analyze_model_efficiency(metrics))
+                emit_efficiency_metrics(metrics)
+                return {'statusCode': 200, 'body': json.dumps({'metrics_collected': len(metrics), 'carbon_tracked': TRACK_CARBON, 'energy_tracked': TRACK_ENERGY})}
+
+            def collect_efficiency_metrics():
+                metrics = {'cpu_utilization': 50.0, 'memory_utilization': 60.0}
+                metrics.update(get_training_metrics())
+                metrics.update(get_model_metrics())
+                return metrics
+
+            def get_training_metrics():
+                return {'epochs_completed': int(os.environ.get('EPOCH', 0)), 'batch_size': int(os.environ.get('BATCH_SIZE', 32)), 'learning_rate': float(os.environ.get('LR', 0.001)), 'training_samples_per_second': float(os.environ.get('SAMPLES_PER_SEC', 0))}
+
+            def get_model_metrics():
+                params = {'computer_vision': 25_000_000, 'natural_language': 110_000_000, 'generative_ai': 1_500_000_000}
+                flops = {'computer_vision': 4_000_000_000, 'natural_language': 20_000_000_000, 'generative_ai': 100_000_000_000}
+                return {'model_parameters': params.get(MODEL_TYPE, 10_000_000), 'flops_per_sample': flops.get(MODEL_TYPE, 1_000_000_000)}
+
+            def calculate_carbon_emissions(metrics):
+                power = estimate_gpu_power()
+                region = os.environ.get('AWS_REGION', 'us-east-1')
+                intensity = get_carbon_intensity(region)
+                hours = metrics.get('epochs_completed', 1) * 0.1
+                emissions = (power * hours * intensity) / 1000
+                result = {'carbon_emissions_gco2': emissions, 'carbon_intensity': intensity, 'power_consumption_watts': power}
+                if metrics.get('model_parameters', 0) > 0:
+                    result['gco2_per_million_parameters'] = emissions / (metrics['model_parameters'] / 1_000_000)
+                return result
+
+            def estimate_gpu_power():
+                power_map = {'ml.p4d.24xlarge': 400, 'ml.p3.16xlarge': 300, 'ml.p3.8xlarge': 250, 'ml.g5.48xlarge': 350, 'ml.g4dn.12xlarge': 200, 'ml.trn1.32xlarge': 300}
+                return power_map.get(os.environ.get('INSTANCE_TYPE', 'ml.p3.8xlarge'), 250)
+
+            def get_carbon_intensity(region):
+                carbon_map = {'us-west-2': 50, 'eu-north-1': 40, 'ca-central-1': 30, 'eu-west-1': 80, 'us-east-1': 400, 'us-east-2': 450, 'eu-central-1': 350, 'ap-southeast-1': 600}
+                return carbon_map.get(region, 400)
+
+            def calculate_energy_usage(metrics):
+                power = metrics.get('power_consumption_watts', 250)
+                hours = metrics.get('epochs_completed', 1) * 0.1
+                energy = (power * hours) / 1000
+                result = {'energy_consumption_kwh': energy}
+                if metrics.get('model_parameters', 0) > 0:
+                    result['kwh_per_billion_parameters'] = energy / (metrics['model_parameters'] / 1_000_000_000)
+                baseline = hours * 300 / 1000
+                result['energy_efficiency_vs_baseline'] = (baseline - energy) / baseline * 100 if baseline > 0 else 0
+                return result
+
+            def analyze_model_efficiency(metrics):
+                analysis = {}
+                if 'gpu_utilization' in metrics:
+                    analysis['gpu_efficiency_score'] = metrics['gpu_utilization']
+                    if metrics['gpu_utilization'] < 70:
+                        analysis['gpu_bottleneck'] = 'underutilized'
+                    elif metrics.get('gpu_memory_used', 0) / max(metrics.get('gpu_memory_total', 1), 1) > 0.9:
+                        analysis['gpu_bottleneck'] = 'memory_bound'
+                    else:
+                        analysis['gpu_bottleneck'] = 'none'
+                analysis['model_efficiency_rating'] = rate_model_efficiency(metrics)
+                analysis['recommendations'] = generate_recommendations(metrics, analysis)
+                return analysis
+
+            def rate_model_efficiency(metrics):
+                score = 100
+                if metrics.get('gpu_utilization', 100) < 80:
+                    score -= 20
+                if metrics.get('energy_efficiency_vs_baseline', 0) < 0:
+                    score -= 10
+                if MODEL_TYPE == 'generative_ai' and metrics.get('model_parameters', 0) > 1e9:
+                    score -= 10
+                return max(0, score)
+
+            def generate_recommendations(metrics, analysis):
+                recs = []
+                if analysis.get('gpu_bottleneck') == 'underutilized':
+                    recs.append({'issue': 'GPU underutilized', 'suggestion': 'Increase batch size or use gradient accumulation', 'potential_improvement': '20-30%'})
+                if analysis.get('gpu_bottleneck') == 'memory_bound':
+                    recs.append({'issue': 'GPU memory saturated', 'suggestion': 'Enable gradient checkpointing or use model parallelism', 'potential_improvement': '15-25%'})
+                if metrics.get('carbon_intensity', 0) > 200:
+                    recs.append({'issue': 'High carbon region', 'suggestion': 'Consider migrating to cleaner region like us-west-2', 'potential_improvement': '60-80% carbon reduction'})
+                return recs
+
+            def store_carbon_data(table, carbon_metrics):
+                ts = int(datetime.now().timestamp())
+                table.put_item(Item={'metric_id': f"training-{ts}", 'timestamp': ts, 'carbon_emissions': carbon_metrics['carbon_emissions_gco2'], 'carbon_intensity': carbon_metrics['carbon_intensity'], 'power_consumption': carbon_metrics['power_consumption_watts'], 'efficiency_metrics': json.dumps(carbon_metrics), 'expiration': ts + 86400 * 30})
+
+            def emit_efficiency_metrics(metrics):
+                namespace = f"SustainableML/{os.environ.get('COMPONENT_NAME', 'default')}"
+                data = [{'MetricName': k, 'Value': v, 'Unit': 'None'} for k, v in metrics.items() if isinstance(v, (int, float))]
+                for i in range(0, len(data), 20):
+                    cloudwatch.put_metric_data(Namespace=namespace, MetricData=data[i:i+20])
+          PYTHON
+        end
+      end
+    end
+  end
+end

data/lib/pangea/components/sustainable_ml_training/modules/code_generators/training_optimizer.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+# Copyright 2025 The Pangea Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Pangea
+  module Components
+    module SustainableMLTraining
+      # Training optimizer Lambda code generator
+      module TrainingOptimizerCode
+        def generate_training_optimizer_code(_input)
+          <<~PYTHON
+            import json
+            import boto3
+            import os
+            from datetime import datetime, timedelta
+            import numpy as np
+
+            sagemaker = boto3.client('sagemaker')
+            dynamodb = boto3.resource('dynamodb')
+            cloudwatch = boto3.client('cloudwatch')
+
+            STATE_TABLE = os.environ['STATE_TABLE']
+            COMPUTE_OPTIMIZATION = os.environ['COMPUTE_OPTIMIZATION']
+            ENABLE_COMPRESSION = os.environ['ENABLE_COMPRESSION'] == 'True'
+            TARGET_REDUCTION = float(os.environ['TARGET_REDUCTION'])
+            EARLY_STOPPING = os.environ['EARLY_STOPPING'] == 'True'
+            PATIENCE = int(os.environ['PATIENCE'])
+
+            def handler(event, context):
+                job_name = event.get('TrainingJobName')
+                if not job_name:
+                    return {'statusCode': 400, 'body': 'No training job specified'}
+                job_details = sagemaker.describe_training_job(TrainingJobName=job_name)
+                if job_details['TrainingJobStatus'] == 'InProgress':
+                    optimizations = apply_runtime_optimizations(job_details)
+                elif job_details['TrainingJobStatus'] == 'Completed':
+                    optimizations = apply_post_training_optimizations(job_details)
+                else:
+                    optimizations = []
+                emit_optimization_metrics(job_name, optimizations)
+                return {'statusCode': 200, 'body': json.dumps({'job_name': job_name, 'optimizations_applied': len(optimizations), 'details': optimizations})}
+
+            def apply_runtime_optimizations(job_details):
+                optimizations = []
+                if EARLY_STOPPING and should_stop_early(job_details):
+                    sagemaker.stop_training_job(TrainingJobName=job_details['TrainingJobName'])
+                    optimizations.append({'type': 'early_stopping', 'reason': 'No improvement in validation metric', 'savings': estimate_carbon_savings(job_details)})
+                if can_adjust_batch_size(job_details):
+                    optimizations.append({'type': 'batch_size_optimization', 'new_value': calculate_optimal_batch_size(job_details), 'efficiency_gain': '15%'})
+                if should_adjust_learning_rate(job_details):
+                    optimizations.append({'type': 'learning_rate_decay', 'strategy': 'cosine_annealing', 'efficiency_gain': '10%'})
+                return optimizations
+
+            def apply_post_training_optimizations(job_details):
+                optimizations = []
+                if ENABLE_COMPRESSION:
+                    result = compress_model(job_details['ModelArtifacts']['S3ModelArtifacts'])
+                    optimizations.append({'type': 'model_compression', 'technique': COMPUTE_OPTIMIZATION, 'size_reduction': result['reduction'], 'accuracy_impact': result['accuracy_delta']})
+                if should_optimize_for_inference(job_details):
+                    result = optimize_for_inference(job_details['ModelArtifacts']['S3ModelArtifacts'])
+                    optimizations.append({'type': 'inference_optimization', 'format': result['format'], 'speedup': result['speedup']})
+                return optimizations
+
+            def should_stop_early(job_details):
+                metrics = get_training_metrics(job_details['TrainingJobName'])
+                if len(metrics) < PATIENCE:
+                    return False
+                return min(metrics[-PATIENCE:]) >= min(metrics[:-PATIENCE])
+
+            def can_adjust_batch_size(job_details):
+                gpu_metrics = get_gpu_metrics(job_details['TrainingJobName'])
+                return gpu_metrics and gpu_metrics['memory_utilization'] < 0.8
+
+            def calculate_optimal_batch_size(job_details):
+                gpu_metrics = get_gpu_metrics(job_details['TrainingJobName'])
+                current = get_current_batch_size(job_details)
+                scale = 1.0 + (1.0 - gpu_metrics['memory_utilization']) * 0.5
+                return 2 ** round(np.log2(int(current * scale)))
+
+            def should_adjust_learning_rate(job_details):
+                metrics = get_training_metrics(job_details['TrainingJobName'])
+                return len(metrics) >= 10 and np.var(metrics[-10:]) < 0.01
+
+            def compress_model(model_s3_uri):
+                results = {'reduction': 0, 'accuracy_delta': 0}
+                if COMPUTE_OPTIMIZATION == 'quantization':
+                    results = {'reduction': 0.75, 'accuracy_delta': -0.01}
+                elif COMPUTE_OPTIMIZATION == 'pruning':
+                    results = {'reduction': TARGET_REDUCTION, 'accuracy_delta': -0.02}
+                elif COMPUTE_OPTIMIZATION == 'distillation':
+                    results = {'reduction': 0.9, 'accuracy_delta': -0.03}
+                return results
+
+            def should_optimize_for_inference(job_details):
+                return True
+
+            def optimize_for_inference(model_s3_uri):
+                instance_family = get_instance_family()
+                if 'p3' in instance_family or 'p4' in instance_family:
+                    return {'format': 'TensorRT', 'speedup': '3x'}
+                elif 'inf1' in instance_family:
+                    return {'format': 'Neuron', 'speedup': '4x'}
+                return {'format': 'ONNX', 'speedup': '2x'}
+
+            def get_training_metrics(job_name):
+                response = cloudwatch.get_metric_statistics(Namespace='aws/sagemaker/TrainingJobs', MetricName='train:loss', Dimensions=[{'Name': 'TrainingJobName', 'Value': job_name}], StartTime=datetime.now() - timedelta(hours=24), EndTime=datetime.now(), Period=300, Statistics=['Average'])
+                return [p['Average'] for p in response['Datapoints']]
+
+            def get_gpu_metrics(job_name):
+                response = cloudwatch.get_metric_statistics(Namespace='aws/sagemaker/TrainingJobs', MetricName='GPUMemoryUtilization', Dimensions=[{'Name': 'TrainingJobName', 'Value': job_name}], StartTime=datetime.now() - timedelta(minutes=10), EndTime=datetime.now(), Period=60, Statistics=['Average'])
+                return {'memory_utilization': response['Datapoints'][-1]['Average'] / 100} if response['Datapoints'] else None
+
+            def get_current_batch_size(job_details):
+                return int(job_details.get('HyperParameters', {}).get('batch_size', 32))
+
+            def estimate_carbon_savings(job_details):
+                elapsed = (datetime.now() - job_details['TrainingStartTime']).total_seconds() / 3600
+                total = float(job_details.get('StoppingCondition', {}).get('MaxRuntimeInSeconds', 86400)) / 3600
+                saved = total - elapsed
+                carbon = (400 * saved * 400) / 1000
+                return {'hours_saved': saved, 'carbon_saved_kg': carbon / 1000}
+
+            def get_instance_family():
+                return os.environ.get('INSTANCE_TYPE', 'ml.p3.8xlarge')
+
+            def emit_optimization_metrics(job_name, optimizations):
+                namespace = f"SustainableML/{os.environ.get('COMPONENT_NAME', 'default')}"
+                data = [{'MetricName': 'OptimizationsApplied', 'Value': len(optimizations), 'Unit': 'Count', 'Dimensions': [{'Name': 'JobName', 'Value': job_name}]}]
+                for opt in optimizations:
+                    if opt['type'] == 'early_stopping':
+                        data.append({'MetricName': 'CarbonSavedByEarlyStopping', 'Value': opt['savings']['carbon_saved_kg'], 'Unit': 'None'})
+                    elif opt['type'] == 'model_compression':
+                        data.append({'MetricName': 'ModelSizeReduction', 'Value': opt['size_reduction'] * 100, 'Unit': 'Percent'})
+                cloudwatch.put_metric_data(Namespace=namespace, MetricData=data)
+          PYTHON
+        end
+      end
+    end
+  end
+end

data/lib/pangea/components/sustainable_ml_training/modules/code_generators.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+# Copyright 2025 The Pangea Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require_relative "code_generators/carbon_scheduler"
+require_relative "code_generators/training_optimizer"
+require_relative "code_generators/efficiency_monitor"
+
+module Pangea
+  module Components
+    module SustainableMLTraining
+      # Code generators for Lambda functions
+      module CodeGenerators
+        include CarbonSchedulerCode
+        include TrainingOptimizerCode
+        include EfficiencyMonitorCode
+      end
+    end
+  end
+end

data/lib/pangea/components/sustainable_ml_training/modules/compute.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+# Copyright 2025 The Pangea Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "base64"
+
+module Pangea
+  module Components
+    module SustainableMLTraining
+      # Compute resources: Instance profile, spot fleet
+      module Compute
+        def create_instance_profile(input, role)
+          aws_iam_instance_profile(:"#{input.name}-instance-profile", {
+            role: role.name,
+            tags: input.tags
+          })
+        end
+
+        def create_spot_fleet(input, instance_profile)
+          aws_spot_fleet_request(:"#{input.name}-training-fleet", {
+            iam_fleet_role: ref(:aws_iam_role, :"#{input.name}-fleet-role", :arn),
+            target_capacity: 1, # Typically 1 for ML training
+            valid_until: (Time.now + 7 * 24 * 60 * 60).iso8601,
+            terminate_instances_with_expiration: true,
+            instance_interruption_behavior: input.spot_interruption_behavior,
+            launch_specification: build_launch_specifications(input, instance_profile),
+            spot_price: calculate_spot_price(input),
+            allocation_strategy: "lowestPrice"
+          })
+        end
+
+        private
+
+        def build_launch_specifications(input, instance_profile)
+          input.preferred_instance_types.map do |instance_type|
+            {
+              instance_type: instance_type,
+              image_id: get_ml_ami(instance_type),
+              iam_instance_profile: {
+                arn: instance_profile.arn
+              },
+              user_data: Base64.encode64(generate_training_user_data(input)),
+              block_device_mappings: [{
+                device_name: "/dev/xvda",
+                ebs: {
+                  volume_size: 500,
+                  volume_type: "gp3",
+                  delete_on_termination: true
+                }
+              }]
+            }
+          end
+        end
+
+        def generate_training_user_data(input)
+          <<~BASH
+            #!/bin/bash
+            # Install monitoring tools
+            pip install codecarbon tensorboard wandb
+
+            # Configure carbon tracking
+            export CODECARBON_COUNTRY=USA
+            export CODECARBON_REGION=oregon
+
+            # Enable mixed precision if configured
+            if [ "#{input.enable_automatic_mixed_precision}" = "true" ]; then
+              export TF_ENABLE_AUTO_MIXED_PRECISION=1
+              export TORCH_AUTOCAST=1
+            fi
+
+            # Set up efficient data loading
+            export NUM_WORKERS=#{input.num_data_loader_workers}
+            export PREFETCH_FACTOR=2
+
+            # Configure model caching
+            export MODEL_CACHE_DIR=/opt/ml/model_cache
+            export TRANSFORMERS_CACHE=$MODEL_CACHE_DIR
+            export HF_HOME=$MODEL_CACHE_DIR
+
+            # Start carbon monitoring
+            codecarbon monitor --project "#{input.name}" --output /opt/ml/output/carbon_report.csv &
+          BASH
+        end
+      end
+    end
+  end
+end