ignis-collective 0.0.1

data/lib/nvruby/collective/dynamic_optimizer.rb

@@ -0,0 +1,308 @@
+# frozen_string_literal: true
+
+require_relative "health_monitor"
+require_relative "topology"
+
+module Ignis
+  module Collective
+    # Dynamic GPU Optimizer
+    # Consolidates all dynamic optimization strategies for NvCCL
+    #
+    # Features:
+    # - Thermal (heatmap) optimization
+    # - Load balancing
+    # - Power management
+    # - Memory management
+    # - GPU availability/selection
+    # - Scheduling optimization
+    # - Synchronization optimization
+    #
+    # @example Create optimizer and get optimal ring order
+    #   optimizer = DynamicOptimizer.new(device_ids: [0, 1, 2, 3])
+    #   ring_order = optimizer.optimal_ring_order(strategy: :thermal)
+    #
+    class DynamicOptimizer
+      # Optimization strategies
+      STRATEGIES = %i[
+        thermal
+        load
+        power
+        memory
+        availability
+        scheduling
+        synchronization
+        balanced
+      ].freeze
+
+      # @return [Array<Integer>] Device IDs
+      attr_reader :device_ids
+
+      # @return [HealthMonitor] Health monitor instance
+      attr_reader :health_monitor
+
+      # @return [Topology::Matrix] Topology matrix
+      attr_reader :topology
+
+      # @param device_ids [Array<Integer>] GPU device IDs
+      # @param health_monitor [HealthMonitor, nil] Optional health monitor
+      def initialize(device_ids:, health_monitor: nil)
+        @device_ids = device_ids.dup.freeze
+        @health_monitor = health_monitor || HealthMonitor.new(device_ids: device_ids)
+        @topology = Topology::Matrix.new(device_ids)
+        @metrics_cache = {}
+        @cache_ttl_seconds = 1.0
+        @last_cache_time = Time.now - 100
+      end
+
+      # Get optimal ring order based on strategy
+      #
+      # @param strategy [Symbol] Optimization strategy
+      # @return [Array<Integer>] Ordered device IDs
+      def optimal_ring_order(strategy: :balanced)
+        refresh_metrics_if_needed!
+
+        case strategy
+        when :thermal
+          thermal_optimized_order
+        when :load
+          load_balanced_order
+        when :power
+          power_optimized_order
+        when :memory
+          memory_optimized_order
+        when :availability
+          availability_optimized_order
+        when :scheduling
+          scheduling_optimized_order
+        when :synchronization
+          synchronization_optimized_order
+        when :balanced
+          balanced_optimized_order
+        else
+          @device_ids.dup
+        end
+      end
+
+      # Get current GPU metrics for all devices
+      #
+      # @return [Hash<Integer, Hash>] Metrics per device
+      def current_metrics
+        refresh_metrics_if_needed!
+        @metrics_cache.dup
+      end
+
+      # Check if a GPU should be excluded from collective
+      #
+      # @param device_id [Integer] Device ID
+      # @return [Boolean] True if device should be excluded
+      def should_exclude?(device_id)
+        metrics = @metrics_cache[device_id]
+        return true unless metrics
+
+        # Exclude if thermal throttling
+        return true if metrics[:temperature] && metrics[:temperature] > 90
+
+        # Exclude if out of memory
+        return true if metrics[:memory_used_percent] && metrics[:memory_used_percent] > 95
+
+        # Exclude if marked unhealthy
+        return true if metrics[:healthy] == false
+
+        false
+      end
+
+      # Get list of available GPUs
+      #
+      # @return [Array<Integer>] Available device IDs
+      def available_devices
+        refresh_metrics_if_needed!
+        @device_ids.reject { |id| should_exclude?(id) }
+      end
+
+      # Suggest optimal chunk size based on memory
+      #
+      # @param total_size [Integer] Total data size in bytes
+      # @return [Integer] Recommended chunk size
+      def suggest_chunk_size(total_size)
+        refresh_metrics_if_needed!
+
+        # Find minimum available memory across all devices
+        min_available = @metrics_cache.values.map do |m|
+          m[:memory_free] || Float::INFINITY
+        end.min
+
+        # Use at most 25% of minimum available memory
+        max_chunk = (min_available * 0.25).to_i
+
+        # Align to 256 bytes
+        chunk = [max_chunk, total_size].min
+        (chunk / 256) * 256
+      end
+
+      # Get power-aware throttling recommendation
+      #
+      # @return [Hash] Power recommendations
+      def power_recommendation
+        refresh_metrics_if_needed!
+
+        high_power_devices = @metrics_cache.select do |_id, m|
+          m[:power_usage] && m[:power_limit] && 
+            (m[:power_usage].to_f / m[:power_limit]) > 0.9
+        end.keys
+
+        {
+          throttle_devices: high_power_devices,
+          should_throttle: high_power_devices.any?,
+          recommendation: high_power_devices.any? ? :reduce_batch_size : :continue
+        }
+      end
+
+      # @return [String]
+      def to_s
+        "DynamicOptimizer[#{@device_ids.size} GPUs, strategy=balanced]"
+      end
+
+      private
+
+      def refresh_metrics_if_needed!
+        return if Time.now - @last_cache_time < @cache_ttl_seconds
+
+        @device_ids.each do |device_id|
+          @metrics_cache[device_id] = collect_device_metrics(device_id)
+        end
+        @last_cache_time = Time.now
+      end
+
+      def collect_device_metrics(device_id)
+        CUDA::RuntimeAPI.cudaSetDevice(device_id)
+
+        # Memory info
+        free_ptr = FFI::MemoryPointer.new(:size_t)
+        total_ptr = FFI::MemoryPointer.new(:size_t)
+        CUDA::RuntimeAPI.cudaMemGetInfo(free_ptr, total_ptr)
+
+        free = free_ptr.read(:size_t)
+        total = total_ptr.read(:size_t)
+        used = total - free
+
+        # Get temperature and power from NVML if available
+        temp = query_temperature(device_id)
+        power = query_power(device_id)
+        power_limit = query_power_limit(device_id)
+
+        {
+          device_id: device_id,
+          memory_free: free,
+          memory_total: total,
+          memory_used: used,
+          memory_used_percent: (used.to_f / total * 100).round(1),
+          temperature: temp,
+          power_usage: power,
+          power_limit: power_limit,
+          healthy: @health_monitor.healthy?(device_id),
+          timestamp: Time.now
+        }
+      rescue StandardError => e
+        Ignis.logger.warn { "Failed to collect metrics for device #{device_id}: #{e.message}" }
+        { device_id: device_id, healthy: false, error: e.message }
+      end
+
+      def query_temperature(device_id)
+        # Use NVML via FFI if available
+        return nil unless defined?(NVML) && NVML.respond_to?(:get_temperature)
+
+        NVML.get_temperature(device_id)
+      rescue StandardError
+        nil
+      end
+
+      def query_power(device_id)
+        return nil unless defined?(NVML) && NVML.respond_to?(:get_power_usage)
+
+        NVML.get_power_usage(device_id)
+      rescue StandardError
+        nil
+      end
+
+      def query_power_limit(device_id)
+        return nil unless defined?(NVML) && NVML.respond_to?(:get_power_limit)
+
+        NVML.get_power_limit(device_id)
+      rescue StandardError
+        nil
+      end
+
+      # Thermal optimization: order by temperature (coolest first)
+      def thermal_optimized_order
+        @device_ids.sort_by do |id|
+          @metrics_cache[id][:temperature] || 0
+        end
+      end
+
+      # Load balancing: order by memory usage (least used first)
+      def load_balanced_order
+        @device_ids.sort_by do |id|
+          @metrics_cache[id][:memory_used_percent] || 0
+        end
+      end
+
+      # Power optimization: order by power usage (lowest first)
+      def power_optimized_order
+        @device_ids.sort_by do |id|
+          @metrics_cache[id][:power_usage] || 0
+        end
+      end
+
+      # Memory optimization: order by free memory (most free first)
+      def memory_optimized_order
+        @device_ids.sort_by do |id|
+          -(@metrics_cache[id][:memory_free] || 0)
+        end
+      end
+
+      # Availability: filter out unhealthy, keep order
+      def availability_optimized_order
+        @device_ids.select { |id| !should_exclude?(id) }
+      end
+
+      # Scheduling: round-robin with healthy bias
+      def scheduling_optimized_order
+        healthy = @device_ids.select { |id| @metrics_cache[id][:healthy] }
+        unhealthy = @device_ids - healthy
+        healthy + unhealthy
+      end
+
+      # Synchronization: order by topology (minimize hops)
+      def synchronization_optimized_order
+        @topology.optimal_ring_order
+      end
+
+      # Balanced: weighted combination of all factors
+      def balanced_optimized_order
+        scores = @device_ids.map do |id|
+          metrics = @metrics_cache[id]
+
+          # Lower is better for each component
+          temp_score = (metrics[:temperature] || 50) / 100.0
+          mem_score = (metrics[:memory_used_percent] || 50) / 100.0
+          power_score = if metrics[:power_usage] && metrics[:power_limit] && metrics[:power_limit] > 0
+                          metrics[:power_usage].to_f / metrics[:power_limit]
+                        else
+                          0.5
+                        end
+          health_score = metrics[:healthy] ? 0.0 : 1.0
+
+          # Weighted sum
+          total = (temp_score * 0.25) +
+                  (mem_score * 0.25) +
+                  (power_score * 0.25) +
+                  (health_score * 0.25)
+
+          [id, total]
+        end
+
+        scores.sort_by { |_id, score| score }.map(&:first)
+      end
+    end
+  end
+end

data/lib/nvruby/collective/health_monitor.rb

@@ -0,0 +1,333 @@
+# frozen_string_literal: true
+
+module Ignis
+  module Collective
+    # GPU health monitoring with proactive failure detection
+    # Monitors heartbeat, memory, and thermal status for each GPU
+    #
+    # @example Usage
+    #   monitor = HealthMonitor.new([0, 1, 2, 3])
+    #   monitor.start!
+    #   
+    #   if monitor.unhealthy_devices.any?
+    #     healer.heal!(monitor.unhealthy_devices)
+    #   end
+    #
+    class HealthMonitor
+      # Health check interval (seconds)
+      HEARTBEAT_INTERVAL = 5.0
+
+      # Minimum free memory percentage
+      MEMORY_THRESHOLD = 0.05
+
+      # Critical temperature (Celsius)
+      TEMP_CRITICAL = 90
+
+      # Warning temperature (Celsius)
+      TEMP_WARNING = 80
+
+      # Consecutive failures before marking unhealthy
+      FAILURE_THRESHOLD = 3
+
+      # Health status values
+      STATUS_HEALTHY = :healthy
+      STATUS_WARNING = :warning
+      STATUS_CRITICAL = :critical
+      STATUS_FAILED = :failed
+
+      # @return [Array<Integer>] Monitored device IDs
+      attr_reader :device_ids
+
+      # @return [Hash<Integer, Hash>] Device health status
+      attr_reader :device_status
+
+      # @return [Boolean] Whether monitoring is active
+      attr_reader :monitoring
+
+      # Create health monitor for specified GPUs
+      #
+      # @param device_ids [Array<Integer>] GPU device IDs to monitor
+      def initialize(device_ids)
+        @device_ids = device_ids.dup.freeze
+        @device_status = {}
+        @monitoring = false
+        @monitor_thread = nil
+        @callbacks = { on_failure: [], on_recovery: [], on_warning: [] }
+
+        initialize_status!
+      end
+
+      # Start background monitoring
+      # @return [void]
+      def start!
+        return if @monitoring
+
+        @monitoring = true
+        @monitor_thread = Thread.new { monitor_loop }
+      end
+
+      # Stop background monitoring
+      # @return [void]
+      def stop!
+        @monitoring = false
+        @monitor_thread&.join(2.0)
+        @monitor_thread = nil
+      end
+
+      # Run single health check (synchronous)
+      # @return [Hash<Integer, Symbol>] Device status map
+      def check_now!
+        @device_ids.each { |id| check_device!(id) }
+        status_summary
+      end
+
+      # Check if specific device is healthy
+      #
+      # @param device_id [Integer] GPU device ID
+      # @return [Boolean] True if healthy
+      def healthy?(device_id)
+        status = @device_status[device_id]
+        return false unless status
+
+        status[:status] == STATUS_HEALTHY || status[:status] == STATUS_WARNING
+      end
+
+      # Get all healthy device IDs
+      # @return [Array<Integer>] Healthy devices
+      def healthy_devices
+        @device_ids.select { |id| healthy?(id) }
+      end
+
+      # Get all unhealthy device IDs
+      # @return [Array<Integer>] Unhealthy devices
+      def unhealthy_devices
+        @device_ids.reject { |id| healthy?(id) }
+      end
+
+      # Get status summary
+      # @return [Hash<Integer, Symbol>] Device → status
+      def status_summary
+        @device_status.transform_values { |v| v[:status] }
+      end
+
+      # Register callback for device failure
+      #
+      # @yield [device_id, reason] Called when device fails
+      # @return [void]
+      def on_failure(&block)
+        @callbacks[:on_failure] << block
+      end
+
+      # Register callback for device recovery
+      #
+      # @yield [device_id] Called when device recovers
+      # @return [void]
+      def on_recovery(&block)
+        @callbacks[:on_recovery] << block
+      end
+
+      # Register callback for warnings
+      #
+      # @yield [device_id, warning_type, value] Called on warning
+      # @return [void]
+      def on_warning(&block)
+        @callbacks[:on_warning] << block
+      end
+
+      # Get detailed status for a device
+      #
+      # @param device_id [Integer] GPU device ID
+      # @return [Hash] Status details
+      def device_details(device_id)
+        @device_status[device_id]&.dup
+      end
+
+      # Force device status (for testing or manual override)
+      #
+      # @param device_id [Integer] GPU device ID
+      # @param status [Symbol] New status
+      # @return [void]
+      def force_status!(device_id, status)
+        return unless @device_status.key?(device_id)
+
+        @device_status[device_id][:status] = status
+        @device_status[device_id][:forced] = true
+      end
+
+      # Clean up resources
+      # @return [void]
+      def destroy!
+        stop!
+        @device_status.clear
+        @callbacks.each_value(&:clear)
+      end
+
+      # @return [String] Human-readable summary
+      def to_s
+        healthy = healthy_devices.size
+        total = @device_ids.size
+        "HealthMonitor[#{healthy}/#{total} healthy]"
+      end
+
+      private
+
+      def initialize_status!
+        @device_ids.each do |id|
+          @device_status[id] = {
+            status: STATUS_HEALTHY,
+            consecutive_failures: 0,
+            last_check: nil,
+            last_healthy: Time.now,
+            memory_free_pct: 1.0,
+            temperature: 0,
+            forced: false
+          }
+        end
+      end
+
+      def monitor_loop
+        while @monitoring
+          @device_ids.each { |id| check_device!(id) }
+          sleep(HEARTBEAT_INTERVAL)
+        end
+      end
+
+      def check_device!(device_id)
+        status = @device_status[device_id]
+        return if status[:forced]
+
+        begin
+          # 1. Heartbeat - synchronize device
+          heartbeat_ok = check_heartbeat(device_id)
+
+          # 2. Memory check
+          memory_pct = check_memory(device_id)
+          memory_ok = memory_pct >= MEMORY_THRESHOLD
+
+          # 3. Temperature check (if NVML available)
+          temp = check_temperature(device_id)
+          temp_ok = temp < TEMP_CRITICAL
+
+          status[:last_check] = Time.now
+          status[:memory_free_pct] = memory_pct
+          status[:temperature] = temp
+
+          if heartbeat_ok && memory_ok && temp_ok
+            # Recovery from failure
+            if status[:status] == STATUS_FAILED
+              notify_recovery(device_id)
+            end
+
+            if temp > TEMP_WARNING
+              status[:status] = STATUS_WARNING
+              notify_warning(device_id, :temperature, temp)
+            elsif memory_pct < MEMORY_THRESHOLD * 2
+              status[:status] = STATUS_WARNING
+              notify_warning(device_id, :memory, memory_pct)
+            else
+              status[:status] = STATUS_HEALTHY
+            end
+
+            status[:consecutive_failures] = 0
+            status[:last_healthy] = Time.now
+          else
+            status[:consecutive_failures] += 1
+
+            if status[:consecutive_failures] >= FAILURE_THRESHOLD
+              old_status = status[:status]
+              status[:status] = STATUS_FAILED
+
+              if old_status != STATUS_FAILED
+                reason = determine_failure_reason(heartbeat_ok, memory_ok, temp_ok, memory_pct, temp)
+                notify_failure(device_id, reason)
+              end
+            else
+              status[:status] = STATUS_CRITICAL
+            end
+          end
+        rescue StandardError => e
+          # Check itself failed
+          status[:consecutive_failures] += 1
+          if status[:consecutive_failures] >= FAILURE_THRESHOLD
+            status[:status] = STATUS_FAILED
+            notify_failure(device_id, "Health check error: #{e.message}")
+          end
+        end
+      end
+
+      def check_heartbeat(device_id)
+        CUDA::RuntimeAPI.ensure_loaded!
+
+        # Use new Fiddle-based RuntimeAPI methods
+        CUDA::RuntimeAPI.set_device(device_id)
+        CUDA::RuntimeAPI.device_synchronize
+        true
+      rescue StandardError
+        false
+      end
+
+      def check_memory(device_id)
+        CUDA::RuntimeAPI.set_device(device_id)
+
+        info = CUDA::RuntimeAPI.mem_get_info
+        free = info[:free_bytes]
+        total = info[:total_bytes]
+
+        total.positive? ? free.to_f / total : 0.0
+      rescue StandardError
+        0.0
+      end
+
+      def check_temperature(device_id)
+        # Try NVML if available
+        return 0 unless defined?(CUDA::NVML)
+
+        CUDA::NVML.device_temperature(device_id)
+      rescue StandardError
+        0  # Assume OK if NVML not available
+      end
+
+      def determine_failure_reason(heartbeat_ok, memory_ok, temp_ok, memory_pct, temp)
+        reasons = []
+        reasons << "heartbeat timeout" unless heartbeat_ok
+        reasons << "out of memory (#{(memory_pct * 100).round(1)}% free)" unless memory_ok
+        reasons << "overheating (#{temp}°C)" unless temp_ok
+        reasons.join(", ")
+      end
+
+      def notify_failure(device_id, reason)
+        @callbacks[:on_failure].each { |cb| cb.call(device_id, reason) }
+
+        # Publish to EventBus for system-wide notification
+        if defined?(Ignis::Shared::EventBus)
+          Ignis::Shared::EventBus.publish(:gpu_failed, payload: {
+            gpu_id: device_id,
+            reason: reason
+          })
+        end
+      end
+
+      def notify_recovery(device_id)
+        @callbacks[:on_recovery].each { |cb| cb.call(device_id) }
+
+        if defined?(Ignis::Shared::EventBus)
+          Ignis::Shared::EventBus.publish(:gpu_recovered, payload: {
+            gpu_id: device_id
+          })
+        end
+      end
+
+      def notify_warning(device_id, warning_type, value)
+        @callbacks[:on_warning].each { |cb| cb.call(device_id, warning_type, value) }
+
+        if defined?(Ignis::Shared::EventBus)
+          Ignis::Shared::EventBus.publish(:health_alert, payload: {
+            gpu_id: device_id,
+            metric: warning_type,
+            value: value
+          })
+        end
+      end
+    end
+  end
+end