easy_ml 0.2.0.pre.rc56 → 0.2.0.pre.rc57

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a2eb7d933162cc05c64a1ea7c21c65f9c80283a1ae45f37226282c503607008f
-  data.tar.gz: e25a1b7b1970753ae6f65917943607102ad52ef9f90831de0660563441448975
+  metadata.gz: e52412950fefc02e9b838930f132873c726440ebbc343159504d7d3287a39d05
+  data.tar.gz: 44ff18d1f1df78b542c8e536427189fce63d147e7e86623d219ed9b89c501ca7
 SHA512:
-  metadata.gz: 6fc39e6b2838ab6242df1848411450764b0434b63bb7e4f1cb60151850e794f1d0a71a956b70b7ca78e159003efc6947d7dbaad9010c7c6899a1baeb8c7570b2
-  data.tar.gz: f50ceecc6935fea0c1f82e5b76beaef2e6ee329087d0f7b7739d4a88b4738d9ed37d13acd47bf12d118092c0b66774772dab9a30f60e12c3854360329caacfa9
+  metadata.gz: 1e543781fb426a6fa7fe6ad6f5b7c924bdab38d88ac8ad7288db3a24f683661b3745a6f2176c993899a9f9737af7e54dfa59cc439a71739d3e2d2d2d75714621
+  data.tar.gz: 3f012c5a3126eec7a69c3c11dd45017f7c2ded7a2bfd5e6e70bcaa388000b19e50d19ed15dc6b47786f61b698cc081e915abade7ece544a3c8a14d0a8f5c4696

data/app/models/easy_ml/column.rb

@@ -30,7 +30,6 @@ module EasyML
     validates :name, uniqueness: { scope: :dataset_id }
 
     before_save :ensure_valid_datatype
-    after_create :set_date_column_if_date_splitter
     after_save :handle_date_column_change
     before_save :set_defaults
 
@@ -41,6 +40,18 @@ module EasyML
     scope :datetime, -> { where(datatype: "datetime") }
     scope :date_column, -> { where(is_date_column: true) }
 
+    def columns
+      [name].concat(virtual_columns)
+    end
+
+    def virtual_columns
+      if one_hot?
+        allowed_categories.map { |cat| "#{name}_#{cat}" }
+      else
+        []
+      end
+    end
+
     def datatype=(dtype)
       write_attribute(:datatype, dtype)
       write_attribute(:polars_datatype, dtype)
@@ -88,9 +99,11 @@ module EasyML
     end
 
     def allowed_categories
-      return nil unless one_hot?
+      return [] unless one_hot?
+      stats = dataset.preprocessor.statistics
+      return [] if stats.nil? || stats.blank?
 
-      dataset.preprocessor.statistics.dup.to_h.dig(name.to_sym, :allowed_categories).sort.concat(["other"])
+      stats.dup.to_h.dig(name.to_sym, :allowed_categories).sort.concat(["other"])
     end
 
     def date_column?

data/app/models/easy_ml/column_list.rb

@@ -1,15 +1,15 @@
 module EasyML
   module ColumnList
-    def sync(only_new: false)
+    def sync(delete: true)
       return unless dataset.schema.present?
 
       EasyML::Column.transaction do
         col_names = syncable
         existing_columns = where(name: col_names)
         import_new(col_names, existing_columns)
+        update_existing(existing_columns)
 
-        if !only_new
-          update_existing(existing_columns)
+        if delete
           delete_missing(existing_columns)
         end
 

data/app/models/easy_ml/dataset.rb

@@ -272,10 +272,10 @@ module EasyML
       raw.split_at.present? && raw.split_at < datasource.last_updated_at
     end
 
-    def learn(only_new: false)
+    def learn(delete: true)
       learn_schema
       learn_statistics
-      columns.sync(only_new: only_new)
+      columns.sync(delete: delete)
     end
 
     def refreshing
@@ -398,7 +398,7 @@ module EasyML
 
       # Learn will update columns, so if any features have been added
       # since the last time columns were learned, we should re-learn the schema
-      learn(only_new: true) if idx == 1 && needs_learn?(df)
+      learn(delete: false) if idx == 1 && needs_learn?(df)
       df = apply_column_mask(df, inference: inference) unless all_columns
       raise_on_nulls(df) if inference
       df, = processed.split_features_targets(df, true, target) if split_ys
@@ -515,7 +515,7 @@ module EasyML
     end
 
     def drop_cols
-      @drop_cols ||= preloaded_columns.select(&:hidden).map(&:name)
+      @drop_cols ||= preloaded_columns.select(&:hidden).flat_map(&:columns)
     end
 
     def drop_if_null

data/app/models/easy_ml/model.rb

@@ -354,15 +354,16 @@ module EasyML
       dataset.decode_labels(ys, col: col)
     end
 
-    def evaluate(y_pred: nil, y_true: nil, x_true: nil, evaluator: nil)
+    def evaluate(y_pred: nil, y_true: nil, x_true: nil, evaluator: nil, dataset: nil)
       evaluator ||= self.evaluator
       if y_pred.nil?
         inputs = default_evaluation_inputs
         y_pred = inputs[:y_pred]
         y_true = inputs[:y_true]
         x_true = inputs[:x_true]
+        dataset = inputs[:dataset]
       end
-      EasyML::Core::ModelEvaluator.evaluate(model: self, y_pred: y_pred, y_true: y_true, x_true: x_true, evaluator: evaluator)
+      EasyML::Core::ModelEvaluator.evaluate(model: self, y_pred: y_pred, y_true: y_true, x_true: x_true, dataset: dataset, evaluator: evaluator)
     end
 
     def evaluator
@@ -524,11 +525,13 @@ module EasyML
 
     def default_evaluation_inputs
       x_true, y_true = dataset.test(split_ys: true)
+      ds = dataset.test(all_columns: true)
       y_pred = predict(x_true)
       {
         x_true: x_true,
         y_true: y_true,
         y_pred: y_pred,
+        dataset: ds,
       }
     end
 

data/app/models/easy_ml/models/xgboost/evals_callback.rb

@@ -32,7 +32,7 @@ module EasyML
           false
         end
 
-        def validation_dataset
+        def test_dataset
           if tuner.present?
             [tuner.x_true, tuner.y_true]
           else
@@ -46,11 +46,12 @@ module EasyML
           log_frequency = 10
           if epoch % log_frequency == 0
             model.adapter.external_model = booster
-            x_true, y_true = validation_dataset
+            x_true, y_true = test_dataset
             @preprocessed ||= model.preprocess(x_true)
             y_pred = model.predict(@preprocessed)
+            dataset = model.dataset.test(all_columns: true)
 
-            metrics = model.evaluate(y_pred: y_pred, y_true: y_true, x_true: x_true)
+            metrics = model.evaluate(y_pred: y_pred, y_true: y_true, x_true: x_true, dataset: dataset)
             Wandb.log(metrics)
           end
 

data/lib/easy_ml/core/evaluators/base_evaluator.rb

@@ -32,7 +32,7 @@ module EasyML
         end
 
         # Instance methods that evaluators must implement
-        def evaluate(y_pred: nil, y_true: nil, x_true: nil)
+        def evaluate(y_pred: nil, y_true: nil, x_true: nil, dataset: nil)
           raise NotImplementedError, "#{self.class} must implement #evaluate"
         end
 

data/lib/easy_ml/core/evaluators/classification_evaluators.rb

@@ -5,7 +5,7 @@ module EasyML
         class AccuracyScore
           include BaseEvaluator
 
-          def evaluate(y_pred:, y_true:, x_true: nil)
+          def evaluate(y_pred:, y_true:, x_true: nil, dataset: nil)
             y_pred = Numo::Int32.cast(y_pred)
             y_true = Numo::Int32.cast(y_true)
             y_pred.eq(y_true).count_true.to_f / y_pred.size
@@ -23,7 +23,7 @@ module EasyML
         class PrecisionScore
           include BaseEvaluator
 
-          def evaluate(y_pred:, y_true:, x_true: nil)
+          def evaluate(y_pred:, y_true:, x_true: nil, dataset: nil)
             y_pred = Numo::Int32.cast(y_pred)
             y_true = Numo::Int32.cast(y_true)
             true_positives = (y_pred.eq(1) & y_true.eq(1)).count_true
@@ -45,7 +45,7 @@ module EasyML
         class RecallScore
           include BaseEvaluator
 
-          def evaluate(y_pred:, y_true:, x_true: nil)
+          def evaluate(y_pred:, y_true:, x_true: nil, dataset: nil)
             y_pred = Numo::Int32.cast(y_pred)
             y_true = Numo::Int32.cast(y_true)
             true_positives = (y_pred.eq(1) & y_true.eq(1)).count_true
@@ -65,9 +65,9 @@ module EasyML
         class F1Score
           include BaseEvaluator
 
-          def evaluate(y_pred:, y_true:, x_true: nil)
-            precision = PrecisionScore.new.evaluate(y_pred: y_pred, y_true: y_true)
-            recall = RecallScore.new.evaluate(y_pred: y_pred, y_true: y_true)
+          def evaluate(y_pred:, y_true:, x_true: nil, dataset: nil)
+            precision = PrecisionScore.new.evaluate(y_pred: y_pred, y_true: y_true, dataset: dataset)
+            recall = RecallScore.new.evaluate(y_pred: y_pred, y_true: y_true, dataset: dataset)
             return 0 unless (precision + recall) > 0
 
             2 * (precision * recall) / (precision + recall)
@@ -85,7 +85,7 @@ module EasyML
         class AUC
           include BaseEvaluator
 
-          def evaluate(y_pred:, y_true:, x_true: nil)
+          def evaluate(y_pred:, y_true:, x_true: nil, dataset: nil)
             y_pred = Numo::DFloat.cast(y_pred)
             y_true = Numo::Int32.cast(y_true)
 
@@ -132,8 +132,8 @@ module EasyML
         class ROC_AUC
           include BaseEvaluator
 
-          def evaluate(y_pred:, y_true:, x_true: nil)
-            AUC.new.evaluate(y_pred: y_pred, y_true: y_true)
+          def evaluate(y_pred:, y_true:, x_true: nil, dataset: nil)
+            AUC.new.evaluate(y_pred: y_pred, y_true: y_true, dataset: dataset)
           end
 
           def description

data/lib/easy_ml/core/evaluators/regression_evaluators.rb

@@ -5,7 +5,7 @@ module EasyML
         class MeanAbsoluteError
           include BaseEvaluator
 
-          def evaluate(y_pred:, y_true:, x_true: nil)
+          def evaluate(y_pred:, y_true:, x_true: nil, dataset: nil)
             (Numo::DFloat.cast(y_pred) - Numo::DFloat.cast(y_true)).abs.mean
           end
 
@@ -21,7 +21,7 @@ module EasyML
         class MeanSquaredError
           include BaseEvaluator
 
-          def evaluate(y_pred:, y_true:, x_true: nil)
+          def evaluate(y_pred:, y_true:, x_true: nil, dataset: nil)
             ((Numo::DFloat.cast(y_pred) - Numo::DFloat.cast(y_true)) ** 2).mean
           end
 
@@ -37,7 +37,7 @@ module EasyML
         class RootMeanSquaredError
           include BaseEvaluator
 
-          def evaluate(y_pred:, y_true:, x_true: nil)
+          def evaluate(y_pred:, y_true:, x_true: nil, dataset: nil)
             Math.sqrt(((Numo::DFloat.cast(y_pred) - Numo::DFloat.cast(y_true)) ** 2).mean)
           end
 
@@ -61,7 +61,7 @@ module EasyML
             "maximize"
           end
 
-          def evaluate(y_pred:, y_true:, x_true: nil)
+          def evaluate(y_pred:, y_true:, x_true: nil, dataset: nil)
             y_true = Numo::DFloat.cast(y_true)
             y_pred = Numo::DFloat.cast(y_pred)
 

data/lib/easy_ml/core/model_evaluator.rb

@@ -98,13 +98,21 @@ module EasyML
           end
         end
 
-        def evaluate(model:, y_pred:, y_true:, x_true: nil, evaluator: nil)
+        def evaluate(model:, y_pred:, y_true:, x_true: nil, evaluator: nil, dataset: nil)
           y_pred = normalize_input(y_pred)
           y_true = normalize_input(y_true)
           check_size(y_pred, y_true)
 
           metrics_results = {}
 
+          if x_true.nil?
+            x_true = model.dataset.test
+          end
+
+          if dataset.nil?
+            dataset = model.dataset.test(all_columns: true)
+          end
+
           model.metrics.each do |metric|
             evaluator_class = get(metric.to_sym)
             next unless evaluator_class
@@ -115,6 +123,7 @@ module EasyML
               y_pred: y_pred,
               y_true: y_true,
               x_true: x_true,
+              dataset: dataset,
             )
           end
 
@@ -124,7 +133,7 @@ module EasyML
             raise "Unknown evaluator: #{evaluator}" unless evaluator_class
 
             evaluator_instance = evaluator_class.new
-            response = evaluator_instance.evaluate(y_pred: y_pred, y_true: y_true, x_true: x_true)
+            response = evaluator_instance.evaluate(y_pred: y_pred, y_true: y_true, x_true: x_true, dataset: dataset)
 
             if response.is_a?(Hash)
               metrics_results.merge!(response)
@@ -145,6 +154,9 @@ module EasyML
         def normalize_input(input)
           case input
           when Array
+            if input.first.class == TrueClass || input.first.class == FalseClass
+              input = input.map { |value| value ? 1.0 : 0.0 }
+            end
             Numo::DFloat.cast(input)
           when Polars::DataFrame
             if input.columns.count > 1
@@ -152,7 +164,10 @@ module EasyML
             end
 
             normalize_input(input[input.columns.first])
-          when Polars::Series, Array
+          when Polars::Series
+            if input.dtype == Polars::Boolean
+              input = input.cast(Polars::Int64)
+            end
             Numo::DFloat.cast(input)
           else
             raise ArgumentError, "Don't know how to evaluate model with y_pred type #{input.class}"

data/lib/easy_ml/core/tuner.rb

@@ -8,7 +8,7 @@ module EasyML
                     :metrics, :objective, :n_trials, :direction, :evaluator,
                     :study, :results, :adapter, :tune_started_at, :x_true, :y_true,
                     :project_name, :job, :current_run, :trial_enumerator, :progress_block,
-                    :tuner_job
+                    :tuner_job, :dataset
 
       def initialize(options = {})
         @model = options[:model]
@@ -77,6 +77,7 @@ module EasyML
         x_true, y_true = model.dataset.test(split_ys: true)
         self.x_true = x_true
         self.y_true = y_true
+        self.dataset = model.dataset.test(all_columns: true)
         adapter.tune_started_at = tune_started_at
         adapter.y_true = y_true
         adapter.x_true = x_true
@@ -96,14 +97,6 @@ module EasyML
             run_metrics = tune_once
             result = calculate_result(run_metrics)
             @results.push(result)
-
-            params = {
-              hyperparameters: model.hyperparameters.to_h,
-              value: result,
-              status: :success,
-            }.compact
-
-            @tuner_run.update!(params)
             @study.tell(@current_trial, result)
           rescue StandardError => e
             @tuner_run.update!(status: :failed, hyperparameters: {})
@@ -138,14 +131,27 @@ module EasyML
         )
         self.current_run = @tuner_run
 
-        adapter.run_trial(@current_trial) do |model|
-          model.fit(tuning: true, &progress_block)
-          y_pred = model.predict(x_true)
-          model.metrics = metrics
-          metrics = model.evaluate(y_pred: y_pred, y_true: y_true, x_true: x_true)
-          puts metrics
-          metrics
+        model = adapter.run_trial(@current_trial) do |model|
+          model.tap do
+            model.fit(tuning: true, &progress_block)
+          end
         end
+
+        y_pred = model.predict(x_true)
+        model.metrics = metrics
+        metrics = model.evaluate(y_pred: y_pred, y_true: y_true, x_true: x_true, dataset: dataset)
+        metric = metrics.symbolize_keys.dig(model.evaluator[:metric].to_sym)
+
+        puts metrics
+
+        params = {
+          hyperparameters: model.hyperparameters.to_h,
+          value: metric,
+          status: :success,
+        }.compact
+
+        @tuner_run.update!(params)
+        metrics
       end
 
       private

data/lib/easy_ml/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module EasyML
-  VERSION = "0.2.0-rc56"
+  VERSION = "0.2.0-rc57"
 
   module Version
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: easy_ml
 version: !ruby/object:Gem::Version
-  version: 0.2.0.pre.rc56
+  version: 0.2.0.pre.rc57
 platform: ruby
 authors:
 - Brett Shollenberger