easy_ml 0.1.3 → 0.2.0.pre.rc1

data/config/vite.json

@@ -0,0 +1,15 @@
+{
+  "all": {
+    "sourceCodeDir": "app/frontend",
+    "watchAdditionalPaths": [],
+    "publicOutputDir": "easy-ml"
+  },
+  "development": {
+    "autoBuild": true,
+    "port": 3037
+  },
+  "test": {
+    "autoBuild": true,
+    "publicOutputDir": "vite-test"
+  }
+}

data/lib/easy_ml/configuration.rb

@@ -0,0 +1,64 @@
+require "singleton"
+require_relative "../../app/models/easy_ml/settings"
+
+module EasyML
+  class Configuration
+    include Singleton
+
+    TIMEZONES = [
+      { value: "America/New_York", label: "Eastern Time" },
+      { value: "America/Chicago", label: "Central Time" },
+      { value: "America/Denver", label: "Mountain Time" },
+      { value: "America/Los_Angeles", label: "Pacific Time" },
+    ]
+    KEYS = EasyML::Settings.configuration_attributes
+    LABELER = {
+      timezone: TIMEZONES,
+    }
+
+    KEYS.each do |key|
+      define_method "#{key}=" do |value|
+        db_settings.send("#{key}=", value)
+      end
+
+      define_method key do
+        db_settings.send(key)
+      end
+
+      if LABELER.key?(key.to_sym)
+        define_method "#{key}_label" do
+          LABELER[key].find { |h| h[:value] == send(key) }[:label]
+        end
+      end
+    end
+
+    class << self
+      def configure
+        yield instance
+        instance.db_settings.save
+      end
+
+      KEYS.each do |key|
+        define_method key do
+          instance.send(key)
+        end
+
+        if LABELER.key?(key.to_sym)
+          define_method "#{key}_label" do
+            instance.send("#{key}_label")
+          end
+        end
+      end
+
+      private
+
+      def db_settings
+        instance.db_settings
+      end
+    end
+
+    def db_settings
+      @db_settings ||= EasyML::Settings.first_or_create
+    end
+  end
+end

data/lib/easy_ml/core/evaluators/base_evaluator.rb

@@ -0,0 +1,53 @@
+module EasyML
+  module Core
+    module Evaluators
+      module BaseEvaluator
+        def self.included(base)
+          base.extend(ClassMethods)
+        end
+
+        def direction
+          "minimize"
+        end
+
+        def label
+          key.split("_").join(" ").titleize
+        end
+
+        def to_option
+          EasyML::Option.new(to_h)
+        end
+
+        def to_h
+          {
+            value: key,
+            label: label,
+            direction: direction
+          }
+        end
+
+        def key
+          self.class.name.split("::").last.underscore
+        end
+
+        # Instance methods that evaluators must implement
+        def evaluate(y_pred: nil, y_true: nil, x_true: nil)
+          raise NotImplementedError, "#{self.class} must implement #evaluate"
+        end
+
+        def calculate_result(metrics)
+          metrics.symbolize_keys!
+          metrics[metric.to_sym]
+        end
+
+        module ClassMethods
+          def self.extended(base)
+            class << base
+              attr_accessor :registry
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/easy_ml/core/evaluators/classification_evaluators.rb

@@ -0,0 +1,126 @@
+module EasyML
+  module Core
+    module Evaluators
+      module ClassificationEvaluators
+        class AccuracyScore
+          include BaseEvaluator
+
+          def evaluate(y_pred:, y_true:, x_true: nil)
+            y_pred = Numo::Int32.cast(y_pred)
+            y_true = Numo::Int32.cast(y_true)
+            y_pred.eq(y_true).count_true.to_f / y_pred.size
+          end
+
+          def direction
+            "maximize"
+          end
+        end
+
+        class PrecisionScore
+          include BaseEvaluator
+
+          def evaluate(y_pred:, y_true:, x_true: nil)
+            y_pred = Numo::Int32.cast(y_pred)
+            y_true = Numo::Int32.cast(y_true)
+            true_positives = (y_pred.eq(1) & y_true.eq(1)).count_true
+            predicted_positives = y_pred.eq(1).count_true
+            return 0 if predicted_positives.zero?
+
+            true_positives.to_f / predicted_positives
+          end
+
+          def direction
+            "maximize"
+          end
+        end
+
+        class RecallScore
+          include BaseEvaluator
+
+          def evaluate(y_pred:, y_true:, x_true: nil)
+            y_pred = Numo::Int32.cast(y_pred)
+            y_true = Numo::Int32.cast(y_true)
+            true_positives = (y_pred.eq(1) & y_true.eq(1)).count_true
+            actual_positives = y_true.eq(1).count_true
+            true_positives.to_f / actual_positives
+          end
+
+          def direction
+            "maximize"
+          end
+        end
+
+        class F1Score
+          include BaseEvaluator
+
+          def evaluate(y_pred:, y_true:, x_true: nil)
+            precision = PrecisionScore.new.evaluate(y_pred: y_pred, y_true: y_true)
+            recall = RecallScore.new.evaluate(y_pred: y_pred, y_true: y_true)
+            return 0 unless (precision + recall) > 0
+
+            2 * (precision * recall) / (precision + recall)
+          end
+
+          def direction
+            "maximize"
+          end
+        end
+
+        class AUC
+          include BaseEvaluator
+
+          def evaluate(y_pred:, y_true:, x_true: nil)
+            y_pred = Numo::DFloat.cast(y_pred)
+            y_true = Numo::Int32.cast(y_true)
+
+            sorted_indices = y_pred.sort_index
+            y_pred[sorted_indices]
+            y_true_sorted = y_true[sorted_indices]
+
+            true_positive_rate = []
+            false_positive_rate = []
+
+            positive_count = y_true_sorted.eq(1).count_true
+            negative_count = y_true_sorted.eq(0).count_true
+
+            tp = 0
+            fp = 0
+
+            y_true_sorted.each do |label|
+              if label == 1
+                tp += 1
+              else
+                fp += 1
+              end
+              true_positive_rate << tp.to_f / positive_count
+              false_positive_rate << fp.to_f / negative_count
+            end
+
+            # Compute the AUC using the trapezoidal rule
+            tpr = Numo::DFloat[*true_positive_rate]
+            fpr = Numo::DFloat[*false_positive_rate]
+
+            auc = ((fpr[1..-1] - fpr[0...-1]) * (tpr[1..-1] + tpr[0...-1]) / 2.0).sum
+            auc
+          end
+
+          def direction
+            "maximize"
+          end
+        end
+
+        class ROC_AUC
+          include BaseEvaluator
+
+          def evaluate(y_pred:, y_true:, x_true: nil)
+            AUC.new.evaluate(y_pred: y_pred, y_true: y_true)
+          end
+
+          def direction
+            "maximize"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/easy_ml/core/evaluators/regression_evaluators.rb

@@ -0,0 +1,66 @@
+module EasyML
+  module Core
+    module Evaluators
+      module RegressionEvaluators
+        class MeanAbsoluteError
+          include BaseEvaluator
+
+          def evaluate(y_pred:, y_true:, x_true: nil)
+            (Numo::DFloat.cast(y_pred) - Numo::DFloat.cast(y_true)).abs.mean
+          end
+
+          def direction
+            "minimize"
+          end
+        end
+
+        class MeanSquaredError
+          include BaseEvaluator
+
+          def evaluate(y_pred:, y_true:, x_true: nil)
+            ((Numo::DFloat.cast(y_pred) - Numo::DFloat.cast(y_true)) ** 2).mean
+          end
+
+          def direction
+            "minimize"
+          end
+        end
+
+        class RootMeanSquaredError
+          include BaseEvaluator
+
+          def evaluate(y_pred:, y_true:, x_true: nil)
+            Math.sqrt(((Numo::DFloat.cast(y_pred) - Numo::DFloat.cast(y_true)) ** 2).mean)
+          end
+
+          def direction
+            "minimize"
+          end
+        end
+
+        class R2Score
+          include BaseEvaluator
+
+          def direction
+            "maximize"
+          end
+
+          def evaluate(y_pred:, y_true:, x_true: nil)
+            y_true = Numo::DFloat.cast(y_true)
+            y_pred = Numo::DFloat.cast(y_pred)
+
+            mean_y = y_true.mean
+            ss_tot = ((y_true - mean_y) ** 2).sum
+            ss_res = ((y_true - y_pred) ** 2).sum
+
+            if ss_tot.zero?
+              ss_res.zero? ? 1.0 : Float::NAN
+            else
+              1 - (ss_res / ss_tot)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/easy_ml/core/model_evaluator.rb

@@ -1,78 +1,86 @@
+require "numo/narray"
+require_relative "evaluators/base_evaluator"
+require_relative "evaluators/regression_evaluators"
+require_relative "evaluators/classification_evaluators"
+
 module EasyML
   module Core
     class ModelEvaluator
-      require "numo/narray"
-
-      EVALUATORS = {
-        mean_absolute_error: lambda { |y_pred, y_true|
-          (Numo::DFloat.cast(y_pred) - Numo::DFloat.cast(y_true)).abs.mean
-        },
-        mean_squared_error: lambda { |y_pred, y_true|
-          ((Numo::DFloat.cast(y_pred) - Numo::DFloat.cast(y_true))**2).mean
-        },
-        root_mean_squared_error: lambda { |y_pred, y_true|
-          Math.sqrt(((Numo::DFloat.cast(y_pred) - Numo::DFloat.cast(y_true))**2).mean)
-        },
-        r2_score: lambda { |y_pred, y_true|
-          # Convert inputs to Numo::DFloat for numerical operations
-          y_true = Numo::DFloat.cast(y_true)
-          y_pred = Numo::DFloat.cast(y_pred)
-
-          # Calculate the mean of the true values
-          mean_y = y_true.mean
-
-          # Calculate Total Sum of Squares (SS_tot)
-          ss_tot = ((y_true - mean_y)**2).sum
-
-          # Calculate Residual Sum of Squares (SS_res)
-          ss_res = ((y_true - y_pred)**2).sum
-
-          # Handle the edge case where SS_tot is zero
-          if ss_tot.zero?
-            if ss_res.zero?
-              # Perfect prediction when both SS_tot and SS_res are zero
-              1.0
-            else
-              # Undefined R² when SS_tot is zero but SS_res is not
-              Float::NAN
+      class << self
+        def callbacks=(callback)
+          @callbacks ||= []
+          @callbacks.push(callback)
+        end
+
+        def callbacks
+          @callbacks || []
+        end
+
+        def register(metric_name, evaluator, type, aliases = {})
+          @registry ||= {}
+          unless evaluator.included_modules.include?(Evaluators::BaseEvaluator)
+            evaluator.include(Evaluators::BaseEvaluator)
+          end
+
+          callbacks.each do |callback|
+            callback.call(metric_name)
+          end
+
+          @registry[metric_name.to_sym] = {
+            evaluator: evaluator,
+            type: type,
+            aliases: (aliases || []).map(&:to_sym),
+          }
+        end
+
+        def get(name)
+          return if name.nil?
+
+          @registry ||= {}
+          option = (@registry[name.to_sym] || @registry.detect do |_k, opts|
+            opts[:aliases].include?(name.to_sym)
+          end.last) || {}
+          option.dig(:evaluator)
+        end
+
+        def for_frontend(evaluator)
+          evaluator.new.to_h
+        end
+
+        def default_evaluator(task)
+          {
+            classification: {
+              metric: "accuracy_score",
+              threshold: 0.70,
+              direction: "maximize",
+            },
+            regression: {
+              metric: "root_mean_squared_error",
+              threshold: 10,
+              direction: "minimize",
+            },
+          }[task.to_sym]
+        end
+
+        def metrics_by_task
+          @registry.group_by { |_key, metric| metric[:type] }.transform_values do |group|
+            group.flat_map do |metric|
+              for_frontend(metric.last.dig(:evaluator))
             end
+          end
+        end
+
+        def metrics(task = nil)
+          if task.nil?
+            @registry.keys
           else
-            # Calculate R²
-            1 - (ss_res / ss_tot)
+            @registry.select do |_k, v|
+              v[:type].to_sym == task.to_sym
+            end.keys
           end
-        },
-        accuracy_score: lambda { |y_pred, y_true|
-          y_pred = Numo::Int32.cast(y_pred)
-          y_true = Numo::Int32.cast(y_true)
-          y_pred.eq(y_true).count_true.to_f / y_pred.size
-        },
-        precision_score: lambda { |y_pred, y_true|
-          y_pred = Numo::Int32.cast(y_pred)
-          y_true = Numo::Int32.cast(y_true)
-          true_positives = (y_pred.eq(1) & y_true.eq(1)).count_true
-          predicted_positives = y_pred.eq(1).count_true
-          return 0 if predicted_positives == 0
-
-          true_positives.to_f / predicted_positives
-        },
-        recall_score: lambda { |y_pred, y_true|
-          y_pred = Numo::Int32.cast(y_pred)
-          y_true = Numo::Int32.cast(y_true)
-          true_positives = (y_pred.eq(1) & y_true.eq(1)).count_true
-          actual_positives = y_true.eq(1).count_true
-          true_positives.to_f / actual_positives
-        },
-        f1_score: lambda { |y_pred, y_true|
-          precision = EVALUATORS[:precision_score].call(y_pred, y_true) || 0
-          recall = EVALUATORS[:recall_score].call(y_pred, y_true) || 0
-          return 0 unless (precision + recall) > 0
-
-          2 * (precision * recall) / (precision + recall)
-        }
-      }
+        end
 
-      class << self
-        def evaluate(model: nil, y_pred: nil, y_true: nil, x_true: nil, evaluator: nil)
+        def evaluate(model:, y_pred:, y_true:, x_true: nil, evaluator: nil)
           y_pred = normalize_input(y_pred)
           y_true = normalize_input(y_true)
           check_size(y_pred, y_true)
@@ -80,45 +88,46 @@ module EasyML
           metrics_results = {}
 
           model.metrics.each do |metric|
-            if metric.is_a?(Module) || metric.is_a?(Class)
-              unless metric.respond_to?(:evaluate)
-                raise "Metric #{metric} must respond to #evaluate in order to be used as a custom evaluator"
-              end
-
-              metrics_results[metric.name] = metric.evaluate(y_pred, y_true)
-            elsif EVALUATORS.key?(metric.to_sym)
-              metrics_results[metric.to_sym] =
-                EVALUATORS[metric.to_sym].call(y_pred, y_true)
-            end
+            evaluator_class = get(metric.to_sym)
+            next unless evaluator_class
+
+            evaluator_instance = evaluator_class.new
+
+            metrics_results[metric.to_sym] = evaluator_instance.evaluate(
+              y_pred: y_pred,
+              y_true: y_true,
+              x_true: x_true,
+            )
           end
 
           if evaluator.present?
-            if evaluator.is_a?(Class)
-              response = evaluator.new.evaluate(y_pred: y_pred, y_true: y_true, x_true: x_true)
-            elsif evaluator.respond_to?(:evaluate)
-              response = evaluator.evaluate(y_pred: y_pred, y_true: y_true, x_true: x_true)
-            elsif evaluator.respond_to?(:call)
-              response = evaluator.call(y_pred: y_pred, y_true: y_true, x_true: x_true)
-            else
-              raise "Don't know how to use CustomEvaluator. Must be a class that responds to evaluate or lambda"
-            end
+            evaluator = evaluator.symbolize_keys!
+            evaluator_class = get(evaluator[:metric])
+            raise "Unknown evaluator: #{evaluator}" unless evaluator_class
+
+            evaluator_instance = evaluator_class.new
+            response = evaluator_instance.evaluate(y_pred: y_pred, y_true: y_true, x_true: x_true)
 
             if response.is_a?(Hash)
               metrics_results.merge!(response)
             else
-              metrics_results[:custom] = response
+              metrics_results[evaluator[:metric].to_sym] = response
             end
           end
 
-          metrics_results
+          metrics_results.symbolize_keys
         end
 
+        private
+
         def check_size(y_pred, y_true)
           raise ArgumentError, "Different sizes" if y_true.size != y_pred.size
         end
 
         def normalize_input(input)
           case input
+          when Array
+            Numo::DFloat.cast(input)
           when Polars::DataFrame
             if input.columns.count > 1
               raise ArgumentError, "Don't know how to evaluate input with multiple columns: #{input}"
@@ -135,3 +144,66 @@ module EasyML
     end
   end
 end
+
+# Register default evaluators
+EasyML::Core::ModelEvaluator.register(
+  :mean_absolute_error,
+  EasyML::Core::Evaluators::RegressionEvaluators::MeanAbsoluteError,
+  :regression,
+  %w[mae]
+)
+EasyML::Core::ModelEvaluator.register(
+  :mean_squared_error,
+  EasyML::Core::Evaluators::RegressionEvaluators::MeanSquaredError,
+  :regression,
+  %w[mse]
+)
+EasyML::Core::ModelEvaluator.register(
+  :root_mean_squared_error,
+  EasyML::Core::Evaluators::RegressionEvaluators::RootMeanSquaredError,
+  :regression,
+  %w[rmse]
+)
+
+EasyML::Core::ModelEvaluator.register(
+  :r2_score,
+  EasyML::Core::Evaluators::RegressionEvaluators::R2Score,
+  :regression,
+  %w[r2]
+)
+EasyML::Core::ModelEvaluator.register(
+  :accuracy_score,
+  EasyML::Core::Evaluators::ClassificationEvaluators::AccuracyScore,
+  :classification,
+  %w[accuracy]
+)
+EasyML::Core::ModelEvaluator.register(
+  :precision_score,
+  EasyML::Core::Evaluators::ClassificationEvaluators::PrecisionScore,
+  :classification,
+  %w[precision]
+)
+EasyML::Core::ModelEvaluator.register(
+  :recall_score,
+  EasyML::Core::Evaluators::ClassificationEvaluators::RecallScore,
+  :classification,
+  %w[recall]
+)
+EasyML::Core::ModelEvaluator.register(
+  :f1_score,
+  EasyML::Core::Evaluators::ClassificationEvaluators::F1Score,
+  :classification,
+  %w[f1]
+)
+# EasyML::Core::ModelEvaluator.register(
+#   :auc,
+#   EasyML::Core::Evaluators::ClassificationEvaluators::AUC,
+#   :classification,
+#   %w[auc]
+# )
+# EasyML::Core::ModelEvaluator.register(
+#   :roc_auc,
+#   EasyML::Core::Evaluators::ClassificationEvaluators::ROC_AUC,
+#   :classification,
+#   %w[roc_auc]
+# )

data/lib/easy_ml/core/tuner/adapters/base_adapter.rb

@@ -3,39 +3,43 @@ module EasyML
     class Tuner
       module Adapters
         class BaseAdapter
-          include GlueGun::DSL
+          attr_accessor :config, :project_name, :tune_started_at, :model,
+                        :x_true, :y_true, :metadata, :model
+
+          def initialize(options = {})
+            @model = options[:model]
+            @config = options[:config] || {}
+            @project_name = options[:project_name]
+            @tune_started_at = options[:tune_started_at]
+            @model = options[:model]
+            @x_true = options[:x_true]
+            @y_true = options[:y_true]
+            @metadata = options[:metadata] || {}
+          end
 
           def defaults
             {}
           end
 
-          attribute :model
-          attribute :config, :hash
-          attribute :project_name, :string
-          attribute :tune_started_at
-          attribute :y_true
-          attribute :x_true
-
           def run_trial(trial)
-            config = deep_merge_defaults(self.config.clone)
+            config = deep_merge_defaults(self.config.clone.deep_symbolize_keys)
             suggest_parameters(trial, config)
-            model.fit
             yield model
           end
 
-          def configure_callbacks
-            raise "Subclasses fof Tuner::Adapter::BaseAdapter must define #configure_callbacks"
-          end
-
           def suggest_parameters(trial, config)
-            defaults.keys.each do |param_name|
-              param_value = suggest_parameter(trial, param_name, config)
-              model.hyperparameters.send("#{param_name}=", param_value)
+            config.keys.inject({}) do |hash, param_name|
+              hash.tap do
+                param_value = suggest_parameter(trial, param_name, config)
+                puts "Suggesting #{param_name}: #{param_value}"
+                model.hyperparameters.send("#{param_name}=", param_value)
+                hash[param_name] = param_value
+              end
             end
           end
 
           def deep_merge_defaults(config)
-            defaults.deep_merge(config) do |_key, default_value, config_value|
+            defaults.deep_symbolize_keys.deep_merge(config.deep_symbolize_keys) do |_key, default_value, config_value|
               if default_value.is_a?(Hash) && config_value.is_a?(Hash)
                 default_value.merge(config_value)
               else
@@ -46,12 +50,18 @@ module EasyML
 
           def suggest_parameter(trial, param_name, config)
             param_config = config[param_name]
+            if !param_config.is_a?(Hash)
+              return param_config
+            end
+
             min = param_config[:min]
             max = param_config[:max]
             log = param_config[:log]
 
             if log
               trial.suggest_loguniform(param_name.to_s, min, max)
+            elsif max.is_a?(Integer) && min.is_a?(Integer)
+              trial.suggest_int(param_name.to_s, min, max)
             else
               trial.suggest_uniform(param_name.to_s, min, max)
             end