easy_ml 0.1.1

data/lib/easy_ml/core/model_evaluator.rb

@@ -0,0 +1,137 @@
+module EasyML
+  module Core
+    class ModelEvaluator
+      require "numo/narray"
+
+      EVALUATORS = {
+        mean_absolute_error: lambda { |y_pred, y_true|
+          (Numo::DFloat.cast(y_pred) - Numo::DFloat.cast(y_true)).abs.mean
+        },
+        mean_squared_error: lambda { |y_pred, y_true|
+          ((Numo::DFloat.cast(y_pred) - Numo::DFloat.cast(y_true))**2).mean
+        },
+        root_mean_squared_error: lambda { |y_pred, y_true|
+          Math.sqrt(((Numo::DFloat.cast(y_pred) - Numo::DFloat.cast(y_true))**2).mean)
+        },
+        r2_score: lambda { |y_pred, y_true|
+          # Convert inputs to Numo::DFloat for numerical operations
+          y_true = Numo::DFloat.cast(y_true)
+          y_pred = Numo::DFloat.cast(y_pred)
+
+          # Calculate the mean of the true values
+          mean_y = y_true.mean
+
+          # Calculate Total Sum of Squares (SS_tot)
+          ss_tot = ((y_true - mean_y)**2).sum
+
+          # Calculate Residual Sum of Squares (SS_res)
+          ss_res = ((y_true - y_pred)**2).sum
+
+          # Handle the edge case where SS_tot is zero
+          if ss_tot.zero?
+            if ss_res.zero?
+              # Perfect prediction when both SS_tot and SS_res are zero
+              1.0
+            else
+              # Undefined R² when SS_tot is zero but SS_res is not
+              Float::NAN
+            end
+          else
+            # Calculate R²
+            1 - (ss_res / ss_tot)
+          end
+        },
+        accuracy_score: lambda { |y_pred, y_true|
+          y_pred = Numo::Int32.cast(y_pred)
+          y_true = Numo::Int32.cast(y_true)
+          y_pred.eq(y_true).count_true.to_f / y_pred.size
+        },
+        precision_score: lambda { |y_pred, y_true|
+          y_pred = Numo::Int32.cast(y_pred)
+          y_true = Numo::Int32.cast(y_true)
+          true_positives = (y_pred.eq(1) & y_true.eq(1)).count_true
+          predicted_positives = y_pred.eq(1).count_true
+          return 0 if predicted_positives == 0
+
+          true_positives.to_f / predicted_positives
+        },
+        recall_score: lambda { |y_pred, y_true|
+          y_pred = Numo::Int32.cast(y_pred)
+          y_true = Numo::Int32.cast(y_true)
+          true_positives = (y_pred.eq(1) & y_true.eq(1)).count_true
+          actual_positives = y_true.eq(1).count_true
+          true_positives.to_f / actual_positives
+        },
+        f1_score: lambda { |y_pred, y_true|
+          precision = EVALUATORS[:precision_score].call(y_pred, y_true) || 0
+          recall = EVALUATORS[:recall_score].call(y_pred, y_true) || 0
+          return 0 unless (precision + recall) > 0
+
+          2 * (precision * recall) / (precision + recall)
+        }
+      }
+
+      class << self
+        def evaluate(model: nil, y_pred: nil, y_true: nil, x_true: nil, evaluator: nil)
+          y_pred = normalize_input(y_pred)
+          y_true = normalize_input(y_true)
+          check_size(y_pred, y_true)
+
+          metrics_results = {}
+
+          model.metrics.each do |metric|
+            if metric.is_a?(Module) || metric.is_a?(Class)
+              unless metric.respond_to?(:evaluate)
+                raise "Metric #{metric} must respond to #evaluate in order to be used as a custom evaluator"
+              end
+
+              metrics_results[metric.name] = metric.evaluate(y_pred, y_true)
+            elsif EVALUATORS.key?(metric.to_sym)
+              metrics_results[metric.to_sym] =
+                EVALUATORS[metric.to_sym].call(y_pred, y_true)
+            end
+          end
+
+          if evaluator.present?
+            if evaluator.is_a?(Class)
+              response = evaluator.new.evaluate(y_pred: y_pred, y_true: y_true, x_true: x_true)
+            elsif evaluator.respond_to?(:evaluate)
+              response = evaluator.evaluate(y_pred: y_pred, y_true: y_true, x_true: x_true)
+            elsif evaluator.respond_to?(:call)
+              response = evaluator.call(y_pred: y_pred, y_true: y_true, x_true: x_true)
+            else
+              raise "Don't know how to use CustomEvaluator. Must be a class that responds to evaluate or lambda"
+            end
+
+            if response.is_a?(Hash)
+              metrics_results.merge!(response)
+            else
+              metrics_results[:custom] = response
+            end
+          end
+
+          metrics_results
+        end
+
+        def check_size(y_pred, y_true)
+          raise ArgumentError, "Different sizes" if y_true.size != y_pred.size
+        end
+
+        def normalize_input(input)
+          case input
+          when Polars::DataFrame
+            if input.columns.count > 1
+              raise ArgumentError, "Don't know how to evaluate input with multiple columns: #{input}"
+            end
+
+            normalize_input(input[input.columns.first])
+          when Polars::Series, Array
+            Numo::DFloat.cast(input)
+          else
+            raise ArgumentError, "Don't know how to evaluate model with y_pred type #{input.class}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/easy_ml/core/models/hyperparameters/base.rb

@@ -0,0 +1,34 @@
+module EasyML
+  module Models
+    module Hyperparameters
+      class Base
+        include GlueGun::DSL
+
+        attribute :learning_rate, :float, default: 0.01
+        attribute :max_iterations, :integer, default: 100
+        attribute :batch_size, :integer, default: 32
+        attribute :regularization, :float, default: 0.0001
+
+        def to_h
+          attributes
+        end
+
+        def merge(other)
+          return self if other.nil?
+
+          other_hash = other.is_a?(Hyperparameters) ? other.to_h : other
+          merged_hash = to_h.merge(other_hash)
+          self.class.new(**merged_hash)
+        end
+
+        def [](key)
+          send(key) if respond_to?(key)
+        end
+
+        def []=(key, value)
+          send("#{key}=", value) if respond_to?("#{key}=")
+        end
+      end
+    end
+  end
+end

data/lib/easy_ml/core/models/hyperparameters/xgboost.rb

@@ -0,0 +1,19 @@
+module EasyML
+  module Models
+    module Hyperparameters
+      class XGBoost < Base
+        include GlueGun::DSL
+
+        attribute :learning_rate, :float, default: 0.1
+        attribute :max_depth, :integer, default: 6
+        attribute :n_estimators, :integer, default: 100
+        attribute :booster, :string, default: "gbtree"
+        attribute :objective, :string, default: "reg:squarederror"
+
+        validates :objective,
+                  inclusion: { in: %w[binary:logistic binary:hinge multi:softmax multi:softprob reg:squarederror
+                                      reg:logistic] }
+      end
+    end
+  end
+end

data/lib/easy_ml/core/models/hyperparameters.rb

@@ -0,0 +1,8 @@
+module EasyML
+  module Models
+    module Hyperparameters
+      require_relative "hyperparameters/base"
+      require_relative "hyperparameters/xgboost"
+    end
+  end
+end

data/lib/easy_ml/core/models/xgboost.rb

@@ -0,0 +1,10 @@
+require_relative "xgboost_core"
+module EasyML
+  module Core
+    module Models
+      class XGBoost < EasyML::Core::Model
+        include XGBoostCore
+      end
+    end
+  end
+end

data/lib/easy_ml/core/models/xgboost_core.rb

@@ -0,0 +1,220 @@
+require "wandb"
+module EasyML
+  module Core
+    module Models
+      module XGBoostCore
+        OBJECTIVES = {
+          classification: {
+            binary: %w[binary:logistic binary:hinge],
+            multi_class: %w[multi:softmax multi:softprob]
+          },
+          regression: %w[reg:squarederror reg:logistic]
+        }
+
+        def self.included(base)
+          base.class_eval do
+            attribute :evaluator
+
+            dependency :callbacks, { array: true } do |dep|
+              dep.option :wandb do |opt|
+                opt.set_class Wandb::XGBoostCallback
+                opt.bind_attribute :log_model, default: false
+                opt.bind_attribute :log_feature_importance, default: true
+                opt.bind_attribute :importance_type, default: "gain"
+                opt.bind_attribute :define_metric, default: true
+                opt.bind_attribute :project_name
+              end
+            end
+
+            dependency :hyperparameters do |dep|
+              dep.set_class EasyML::Models::Hyperparameters::XGBoost
+              dep.bind_attribute :batch_size, default: 32
+              dep.bind_attribute :learning_rate, default: 1.1
+              dep.bind_attribute :max_depth, default: 6
+              dep.bind_attribute :n_estimators, default: 100
+              dep.bind_attribute :booster, default: "gbtree"
+              dep.bind_attribute :objective, default: "reg:squarederror"
+            end
+          end
+        end
+
+        attr_accessor :model, :booster
+
+        def predict(xs)
+          raise "No trained model! Train a model before calling predict" unless @booster.present?
+          raise "Cannot predict on nil — XGBoost" if xs.nil?
+
+          y_pred = @booster.predict(preprocess(xs))
+
+          case task.to_sym
+          when :classification
+            to_classification(y_pred)
+          else
+            y_pred
+          end
+        end
+
+        def predict_proba(data)
+          dmat = DMatrix.new(data)
+          y_pred = @booster.predict(dmat)
+
+          if y_pred.first.is_a?(Array)
+            # multiple classes
+            y_pred
+          else
+            y_pred.map { |v| [1 - v, v] }
+          end
+        end
+
+        def load(path = nil)
+          path ||= file
+          path = path&.file&.file if path.class.ancestors.include?(CarrierWave::Uploader::Base)
+
+          raise "No existing model at #{path}" unless File.exist?(path)
+
+          initialize_model do
+            booster_class.new(params: hyperparameters.to_h, model_file: path)
+          end
+        end
+
+        def _save_model_file(path)
+          puts "XGBoost received path #{path}"
+          @booster.save_model(path)
+        end
+
+        def feature_importances
+          @model.booster.feature_names.zip(@model.feature_importances).to_h
+        end
+
+        def base_model
+          ::XGBoost
+        end
+
+        def customize_callbacks
+          yield callbacks
+        end
+
+        private
+
+        def booster_class
+          ::XGBoost::Booster
+        end
+
+        def d_matrix_class
+          ::XGBoost::DMatrix
+        end
+
+        def model_class
+          ::XGBoost::Model
+        end
+
+        def train
+          validate_objective
+
+          xs = xs.to_a.map(&:values)
+          ys = ys.to_a.map(&:values)
+          dtrain = d_matrix_class.new(xs, label: ys)
+          @model = base_model.train(hyperparameters.to_h, dtrain, callbacks: callbacks)
+        end
+
+        def train_in_batches
+          validate_objective
+
+          # Initialize the model with the first batch
+          @model = nil
+          @booster = nil
+          x_valid, y_valid = dataset.valid(split_ys: true)
+          x_train, y_train = dataset.train(split_ys: true)
+          fit_batch(x_train, y_train, x_valid, y_valid)
+        end
+
+        def _preprocess(df)
+          df.to_a.map do |row|
+            row.values.map do |value|
+              case value
+              when Time
+                value.to_i # Convert Time to Unix timestamp
+              when Date
+                value.to_time.to_i # Convert Date to Unix timestamp
+              when String
+                value
+              when TrueClass, FalseClass
+                value ? 1.0 : 0.0 # Convert booleans to 1.0 and 0.0
+              when Integer
+                value
+              else
+                value.to_f # Ensure everything else is converted to a float
+              end
+            end
+          end
+        end
+
+        def preprocess(xs, ys = nil)
+          column_names = xs.columns
+          xs = _preprocess(xs)
+          ys = ys.nil? ? nil : _preprocess(ys).flatten
+          kwargs = { label: ys }.compact
+          ::XGBoost::DMatrix.new(xs, **kwargs).tap do |dmat|
+            dmat.instance_variable_set(:@feature_names, column_names)
+          end
+        end
+
+        def initialize_model
+          @model = model_class.new(n_estimators: @hyperparameters.to_h.dig(:n_estimators))
+          @booster = yield
+          @model.instance_variable_set(:@booster, @booster)
+        end
+
+        def validate_objective
+          objective = hyperparameters.objective
+          unless task.present?
+            raise ArgumentError,
+                  "cannot train model without task. Please specify either regression or classification (model.task = :regression)"
+          end
+
+          case task.to_sym
+          when :classification
+            _, ys = dataset.data(split_ys: true)
+            classification_type = ys[ys.columns.first].uniq.count <= 2 ? :binary : :multi_class
+            allowed_objectives = OBJECTIVES[:classification][classification_type]
+          else
+            allowed_objectives = OBJECTIVES[task.to_sym]
+          end
+          return if allowed_objectives.map(&:to_sym).include?(objective.to_sym)
+
+          raise ArgumentError,
+                "cannot use #{objective} for #{task} task. Allowed objectives are: #{allowed_objectives.join(", ")}"
+        end
+
+        def fit_batch(x_train, y_train, x_valid, y_valid)
+          d_train = preprocess(x_train, y_train)
+          d_valid = preprocess(x_valid, y_valid)
+
+          evals = [[d_train, "train"], [d_valid, "eval"]]
+
+          # # If this is the first batch, create the booster
+          if @booster.nil?
+            initialize_model do
+              base_model.train(@hyperparameters.to_h, d_train,
+                               num_boost_round: @hyperparameters.to_h.dig("n_estimators"), evals: evals, callbacks: callbacks)
+            end
+          else
+            # Update the existing booster with the new batch
+            @model.update(d_train)
+          end
+        end
+
+        def to_classification(y_pred)
+          if y_pred.first.is_a?(Array)
+            # multiple classes
+            y_pred.map do |v|
+              v.map.with_index.max_by { |v2, _| v2 }.last
+            end
+          else
+            y_pred.map { |v| v > 0.5 ? 1 : 0 }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/easy_ml/core/models.rb

@@ -0,0 +1,10 @@
+module EasyML
+  module Core
+    module Models
+      require_relative "models/hyperparameters"
+      require_relative "models/xgboost"
+
+      AVAILABLE_MODELS = [XGBoost]
+    end
+  end
+end

data/lib/easy_ml/core/tuner/adapters/base_adapter.rb

@@ -0,0 +1,63 @@
+module EasyML
+  module Core
+    class Tuner
+      module Adapters
+        class BaseAdapter
+          include GlueGun::DSL
+
+          def defaults
+            {}
+          end
+
+          attribute :model
+          attribute :config, :hash
+          attribute :project_name, :string
+          attribute :tune_started_at
+          attribute :y_true
+          attribute :x_true
+
+          def run_trial(trial)
+            config = deep_merge_defaults(self.config.clone)
+            suggest_parameters(trial, config)
+            model.fit
+            yield model
+          end
+
+          def configure_callbacks
+            raise "Subclasses fof Tuner::Adapter::BaseAdapter must define #configure_callbacks"
+          end
+
+          def suggest_parameters(trial, config)
+            defaults.keys.each do |param_name|
+              param_value = suggest_parameter(trial, param_name, config)
+              model.hyperparameters.send("#{param_name}=", param_value)
+            end
+          end
+
+          def deep_merge_defaults(config)
+            defaults.deep_merge(config) do |_key, default_value, config_value|
+              if default_value.is_a?(Hash) && config_value.is_a?(Hash)
+                default_value.merge(config_value)
+              else
+                config_value
+              end
+            end
+          end
+
+          def suggest_parameter(trial, param_name, config)
+            param_config = config[param_name]
+            min = param_config[:min]
+            max = param_config[:max]
+            log = param_config[:log]
+
+            if log
+              trial.suggest_loguniform(param_name.to_s, min, max)
+            else
+              trial.suggest_uniform(param_name.to_s, min, max)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/easy_ml/core/tuner/adapters/xgboost_adapter.rb

@@ -0,0 +1,50 @@
+require_relative "base_adapter"
+
+module EasyML
+  module Core
+    class Tuner
+      module Adapters
+        class XGBoostAdapter < BaseAdapter
+          include GlueGun::DSL
+
+          def defaults
+            {
+              learning_rate: {
+                min: 0.001,
+                max: 0.1,
+                log: true
+              },
+              n_estimators: {
+                min: 100,
+                max: 1_000
+              },
+              max_depth: {
+                min: 2,
+                max: 20
+              }
+            }
+          end
+
+          def configure_callbacks
+            model.customize_callbacks do |callbacks|
+              return unless callbacks.present?
+
+              wandb_callback = callbacks.detect { |cb| cb.class == Wandb::XGBoostCallback }
+              return unless wandb_callback.present?
+
+              wandb_callback.project_name = "#{wandb_callback.project_name}_#{tune_started_at.strftime("%Y_%m_%d_%H_%M_%S")}"
+              wandb_callback.custom_loggers = [
+                lambda do |booster, _epoch, _hist|
+                  dtrain = model.send(:preprocess, x_true, y_true)
+                  y_pred = booster.predict(dtrain)
+                  metrics = model.evaluate(y_pred: y_pred, y_true: y_true, x_true: x_true)
+                  Wandb.log(metrics)
+                end
+              ]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/easy_ml/core/tuner/adapters.rb

@@ -0,0 +1,10 @@
+module EasyML
+  module Core
+    class Tuner
+      module Adapters
+        require_relative "adapters/base_adapter"
+        require_relative "adapters/xgboost_adapter"
+      end
+    end
+  end
+end

data/lib/easy_ml/core/tuner.rb

@@ -0,0 +1,105 @@
+require "optuna"
+require_relative "tuner/adapters"
+
+module EasyML
+  module Core
+    class Tuner
+      include GlueGun::DSL
+
+      attribute :model
+      attribute :dataset
+      attribute :project_name, :string
+      attribute :task, :string
+      attribute :config, :hash, default: {}
+      attribute :metrics, :array
+      attribute :objective, :string
+      attribute :n_trials, default: 100
+      attribute :callbacks, :array
+      attr_accessor :study, :results
+
+      dependency :adapter, lazy: false do |dep|
+        dep.option :xgboost do |opt|
+          opt.set_class Adapters::XGBoostAdapter
+          opt.bind_attribute :model
+          opt.bind_attribute :config
+          opt.bind_attribute :project_name
+          opt.bind_attribute :tune_started_at
+          opt.bind_attribute :y_true
+        end
+
+        dep.when do |_dep|
+          case model
+          when EasyML::Core::Models::XGBoost, EasyML::Models::XGBoost
+            { option: :xgboost }
+          end
+        end
+      end
+
+      def loggers(_study, trial)
+        return unless trial.state.name == "FAIL"
+
+        raise "Trial failed: Stopping optimization."
+      end
+
+      def tune
+        set_defaults!
+
+        @study = Optuna::Study.new
+        @results = []
+        model.task = task
+        x_true, y_true = model.dataset.test(split_ys: true)
+        tune_started_at = EST.now
+        adapter = pick_adapter.new(model: model, config: config, tune_started_at: tune_started_at, y_true: y_true,
+                                   x_true: x_true)
+        adapter.configure_callbacks
+
+        @study.optimize(n_trials: n_trials, callbacks: [method(:loggers)]) do |trial|
+          run_metrics = tune_once(trial, x_true, y_true, adapter)
+
+          result = if model.evaluator.present?
+                     if model.evaluator_metric.present?
+                       run_metrics[model.evaluator_metric]
+                     else
+                       run_metrics[:custom]
+                     end
+                   else
+                     run_metrics[objective.to_sym]
+                   end
+          @results.push(result)
+          result
+        rescue StandardError => e
+          puts "Optuna failed with: #{e.message}"
+        end
+
+        raise "Optuna study failed" unless @study.respond_to?(:best_trial)
+
+        @study.best_trial.params
+      end
+
+      def pick_adapter
+        case model
+        when EasyML::Core::Models::XGBoost, EasyML::Models::XGBoost
+          Adapters::XGBoostAdapter
+        end
+      end
+
+      def tune_once(trial, x_true, y_true, adapter)
+        adapter.run_trial(trial) do |model|
+          y_pred = model.predict(y_true)
+          model.metrics = metrics
+          model.evaluate(y_pred: y_pred, y_true: y_true, x_true: x_true)
+        end
+      end
+
+      def set_defaults!
+        unless task.present?
+          self.task = model.task
+          raise ArgumentError, "EasyML::Core::Tuner requires task (regression or classification)" unless task.present?
+        end
+        raise ArgumentError, "Objectives required for EasyML::Core::Tuner" unless objective.present?
+
+        self.metrics = EasyML::Core::Model.new(task: task).allowed_metrics if metrics.nil? || metrics.empty?
+      end
+    end
+  end
+end

data/lib/easy_ml/core/uploaders/model_uploader.rb

@@ -0,0 +1,24 @@
+require "carrierwave"
+
+module EasyML
+  module Core
+    module Uploaders
+      class ModelUploader < CarrierWave::Uploader::Base
+        # Choose storage type
+        if Rails.env.production?
+          storage :fog
+        else
+          storage :file
+        end
+
+        def store_dir
+          "easy_ml_models/#{model.name}"
+        end
+
+        def extension_allowlist
+          %w[bin model json]
+        end
+      end
+    end
+  end
+end