easy_ml 0.2.0.pre.rc57 → 0.2.0.pre.rc60

data/app/models/easy_ml/column/imputers/base.rb

@@ -0,0 +1,89 @@
+module EasyML
+  class Column
+    class Imputers
+      class Base
+        class << self
+          def param_applies(p)
+            Imputers.supported_params << p
+            Imputers.params_by_class[self] ||= []
+            Imputers.params_by_class[self] << p.to_sym
+          end
+
+          def method_applies(m)
+            Imputers.supported_methods << m.to_sym
+            Imputers.methods_by_class[self] ||= []
+            Imputers.methods_by_class[self] << m.to_sym
+          end
+
+          def description
+            "Unknown preprocessing method"
+          end
+        end
+
+        attr_accessor :column, :preprocessing_step
+
+        def initialize(column, preprocessing_step)
+          @column = column
+          @preprocessing_step = preprocessing_step.with_indifferent_access
+        end
+
+        def applies?
+          method_applies? || param_applies?
+        end
+
+        def method_applies?
+          imputers_own_methods.include?(method.to_sym)
+        end
+
+        def param_applies?
+          params.keys.any? { |p| imputers_own_params.include?(p.to_sym) && params[p] != false }
+        end
+
+        def imputers_own_methods
+          Imputers.methods_by_class[self.class] || []
+        end
+
+        def imputers_own_params
+          Imputers.params_by_class[self.class] || []
+        end
+
+        def params
+          @preprocessing_step.dig(:params)
+        end
+
+        def method
+          @preprocessing_step.dig(:method)
+        end
+
+        def statistics(*args)
+          if column.is_computed
+            column.statistics.dig(:processed, *args)
+          else
+            column.statistics.dig(:clipped, *args) || column.statistics.dig(:raw, *args)
+          end
+        end
+
+        def anything?
+          true
+        end
+
+        def inspect
+          params_str = params ? params.map { |k, v| "#{k}: #{v}" }.join(", ") : "none"
+          method_str = method ? method : "none"
+
+          "#<#{self.class.name} method=#{method_str.inspect} params={#{params_str}}>"
+        end
+
+        alias_method :to_s, :inspect
+
+        def transform(df)
+          raise "Method not implemented"
+        end
+
+        def description
+          self.class.description
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/imputers/categorical.rb

@@ -0,0 +1,35 @@
+module EasyML
+  class Column
+    class Imputers
+      class Categorical < Base
+        method_applies :categorical
+        param_applies :categorical_min
+
+        def self.description
+          "Categorical imputation"
+        end
+
+        def transform(df)
+          return df unless allowed_categories.present?
+
+          case column.datatype
+          when :categorical
+            df = df.with_column(
+              Polars.when(Polars.col(column.name).is_in(allowed_categories))
+                .then(Polars.col(column.name))
+                .otherwise(Polars.lit("other"))
+                .alias(column.name)
+            )
+          when :boolean
+            # no-op
+          end
+          df
+        end
+
+        def allowed_categories
+          column.allowed_categories
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/imputers/clip.rb

@@ -0,0 +1,30 @@
+module EasyML
+  class Column
+    class Imputers
+      class Clip < Base
+        attr_accessor :column, :dataset, :preprocessing_step
+
+        param_applies :clip
+
+        def self.description
+          "Clip"
+        end
+
+        def transform(df)
+          df = df.with_column(
+            Polars.col(column.name).clip(min, max).alias(column.name)
+          )
+          df
+        end
+
+        def min
+          params.dig(:clip, :min) || 0
+        end
+
+        def max
+          params.dig(:clip, :max) || Float::INFINITY
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/imputers/constant.rb

@@ -0,0 +1,27 @@
+module EasyML
+  class Column
+    class Imputers
+      class Constant < Base
+        method_applies :constant
+        param_applies :constant
+
+        def self.description
+          "Constant value imputation"
+        end
+
+        def transform(df)
+          return df unless constant.present?
+
+          df = df.with_column(
+            Polars.col(column.name).fill_null(constant).alias(column.name)
+          )
+          df
+        end
+
+        def constant
+          params.dig(:constant)
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/imputers/ffill.rb

@@ -0,0 +1,29 @@
+module EasyML
+  class Column
+    class Imputers
+      class Ffill < Base
+        method_applies :ffill
+
+        def self.description
+          "Forward fill imputation"
+        end
+
+        def transform(df)
+          return df unless last_value.present?
+
+          df = df.with_column(
+            Polars.when(Polars.col(column.name).is_null)
+                  .then(Polars.lit(last_value).cast(column.polars_datatype))
+                  .otherwise(Polars.col(column.name).cast(column.polars_datatype))
+                  .alias(column.name)
+          )
+          df
+        end
+
+        def last_value
+          statistics(:last_value)
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/imputers/imputer.rb

@@ -0,0 +1,103 @@
+module EasyML
+  class Column
+    class Imputers
+      class Imputer
+        attr_accessor :dataset, :column, :preprocessing_step
+
+        def initialize(column, preprocessing_step)
+          @column = column
+          @dataset = column.dataset
+          @preprocessing_step = preprocessing_step.with_indifferent_access
+          validate_preprocessing_step!
+        end
+
+        def inspect
+          "#<#{self.class.name} adapters=#{adapters.map(&:inspect).join(", ")}>"
+        end
+
+        def ordered_adapters
+          [
+            Clip,
+            Mean,
+            Median,
+            Constant,
+            Ffill,
+            Categorical,
+            MostFrequent,
+            Today,
+            OneHotEncoder,
+            OrdinalEncoder,
+          ]
+        end
+
+        def adapters
+          @adapters ||= ordered_adapters.map { |klass| klass.new(column, preprocessing_step) }.select(&:applies?)
+        end
+
+        def imputers
+          return nil if column.preprocessing_steps.blank?
+
+          @imputers ||= column.preprocessing_steps.keys.reduce({}) do |hash, key|
+            hash[key.to_sym] = Imputer.new(
+              column: column,
+              preprocessing_step: column.preprocessing_steps[key],
+            )
+          end
+        end
+
+        def description
+          adapters.map(&:description).compact.join(", ")
+        end
+
+        def anything?
+          adapters.any?
+        end
+
+        def transform(df)
+          return df unless anything?
+
+          adapters.reduce(df) do |df, adapter|
+            adapter.transform(df)
+          end
+        end
+
+        def clip(df)
+          return df unless adapters.map(&:class).include?(Clip)
+
+          EasyML::Column::Imputers::Clip.new(column, preprocessing_step).transform(df)
+        end
+
+        def decode_labels(df)
+          return df unless adapters.map(&:class).include?(OrdinalEncoder)
+
+          EasyML::Column::Imputers::OrdinalEncoder.new(column, preprocessing_step).decode_labels(df)
+        end
+
+        private
+
+        def validate_preprocessing_step!
+          validate_params!
+          validate_method!
+        end
+
+        def validate_params!
+          return unless preprocessing_step[:params]
+
+          preprocessing_step[:params].keys.each do |param|
+            unless Imputers.supported_params.include?(param.to_sym)
+              raise ArgumentError, "Unsupported preprocessing parameter '#{param}'. Supported parameters are: #{Imputers.supported_params.join(", ")}"
+            end
+          end
+        end
+
+        def validate_method!
+          return unless preprocessing_step[:method]
+
+          unless Imputers.supported_methods.include?(preprocessing_step[:method].to_sym)
+            raise ArgumentError, "Unsupported preprocessing method '#{preprocessing_step[:method]}'. Supported methods are: #{Imputers.supported_methods.join(", ")}"
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/imputers/mean.rb

@@ -0,0 +1,27 @@
+module EasyML
+  class Column
+    class Imputers
+      class Mean < Base
+        method_applies :mean
+
+        def self.description
+          "Mean imputation"
+        end
+
+        def transform(df)
+          return df unless mean.present?
+
+          mean = statistics(:mean)
+          df = df.with_column(
+            Polars.col(column.name).fill_null(mean).alias(column.name)
+          )
+          df
+        end
+
+        def mean
+          statistics(:mean)
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/imputers/median.rb

@@ -0,0 +1,27 @@
+module EasyML
+  class Column
+    class Imputers
+      class Median < Base
+        method_applies :median
+
+        def self.description
+          "Median imputation"
+        end
+
+        def transform(df)
+          return df unless median.present?
+
+          median = statistics(:median)
+          df = df.with_column(
+            Polars.col(column.name).fill_null(median).alias(column.name)
+          )
+          df
+        end
+
+        def median
+          statistics(:median)
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/imputers/most_frequent.rb

@@ -0,0 +1,27 @@
+module EasyML
+  class Column
+    class Imputers
+      class MostFrequent < Base
+        method_applies :most_frequent
+
+        def self.description
+          "Most frequent value imputation"
+        end
+
+        def transform(df)
+          return df unless most_frequent.present?
+
+          most_frequent = statistics(:most_frequent_value)
+          df = df.with_column(
+            Polars.col(column.name).fill_null(most_frequent).alias(column.name)
+          )
+          df
+        end
+
+        def most_frequent
+          statistics(:most_frequent_value)
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/imputers/null_imputer.rb

@@ -0,0 +1,15 @@
+module EasyML
+  class Column
+    class Imputers
+      class NullImputer
+        def anything?
+          false
+        end
+
+        def method_missing(_name, df)
+          df
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/imputers/one_hot_encoder.rb

@@ -0,0 +1,30 @@
+module EasyML
+  class Column
+    class Imputers
+      class OneHotEncoder < Base
+        param_applies :one_hot
+
+        def self.description
+          "One-hot encoder"
+        end
+
+        def transform(df)
+          return df unless allowed_categories.present?
+
+          allowed_categories.each do |value|
+            new_col_name = "#{column.name}_#{value}".gsub(/-/, "_")
+            df = df.with_column(
+              df[column.name].cast(Polars::String).eq(value.to_s).cast(Polars::Boolean).alias(new_col_name)
+            )
+          end
+          df = df.drop([column.name])
+          df
+        end
+
+        def allowed_categories
+          column.allowed_categories.sort
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/imputers/ordinal_encoder.rb

@@ -0,0 +1,78 @@
+module EasyML
+  class Column
+    class Imputers
+      class OrdinalEncoder < Base
+        param_applies :ordinal_encoding
+
+        def self.description
+          "Ordinal encoder"
+        end
+
+        def transform(df)
+          return df unless label_encoder.present?
+
+          case column.datatype
+          when :categorical
+            df = df.with_column(
+              Polars.when(Polars.col(column.name).is_in(allowed_categories))
+                .then(Polars.col(column.name))
+                .otherwise(Polars.lit("other"))
+                .alias(column.name)
+            )
+          when :boolean
+            # no-op
+          end
+
+          df = df.with_column(
+            df[column.name].map { |v| label_encoder[column.cast(v)] || other_value }.alias(column.name)
+          )
+
+          df
+        end
+
+        def decode_labels(df)
+          if df.is_a?(Array)
+            return df.map { |v| label_decoder[v.to_i] }
+          end
+
+          df = df.with_column(
+            df[column.name].map { |v| label_decoder[v.to_i] }.alias(column.name)
+          )
+          df
+        end
+
+        def categories
+          label_encoder.keys
+        end
+
+        def values
+          label_encoder.values
+        end
+
+        def cast_encoder(encoder)
+          encoder.transform_keys { |k| column.cast(k) }
+        end
+
+        def cast_decoder(decoder)
+          decoder.transform_keys { |k| k.to_i }
+        end
+
+        def label_encoder
+          @label_encoder ||= cast_encoder(statistics(:label_encoder))
+        end
+
+        def label_decoder
+          @label_decoder ||= cast_decoder(statistics(:label_decoder))
+        end
+
+        def other_value
+          label_encoder.values.max + 1
+        end
+
+        def allowed_categories
+          column.allowed_categories
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/imputers/today.rb

@@ -0,0 +1,20 @@
+module EasyML
+  class Column
+    class Imputers
+      class Today < Base
+        method_applies :today
+
+        def self.description
+          "Current date imputation"
+        end
+
+        def transform(df)
+          df = df.with_column(
+            Polars.col(column.name).fill_null(Polars.lit(UTC.today.beginning_of_day)).alias(column.name)
+          )
+          df
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/imputers.rb

@@ -0,0 +1,126 @@
+module EasyML
+  class Column
+    class Imputers
+      attr_accessor :dataset, :column
+
+      ALLOWED_PARAMS = {
+        constant: [:constant],
+        categorical: %i[categorical_min one_hot ordinal_encoding],
+        most_frequent: %i[one_hot ordinal_encoding],
+        mean: [:clip],
+        median: [:clip],
+      }
+
+      PREPROCESSING_STRATEGIES = {
+        float: [
+          { value: "ffill", label: "Forward Fill" },
+          { value: "mean", label: "Mean" },
+          { value: "median", label: "Median" },
+          { value: "constant", label: "Constant Value" },
+        ],
+        integer: [
+          { value: "ffill", label: "Forward Fill" },
+          { value: "mean", label: "Mean" },
+          { value: "median", label: "Median" },
+          { value: "constant", label: "Constant Value" },
+        ],
+        boolean: [
+          { value: "ffill", label: "Forward Fill" },
+          { value: "most_frequent", label: "Most Frequent" },
+          { value: "constant", label: "Constant Value" },
+        ],
+        datetime: [
+          { value: "ffill", label: "Forward Fill" },
+          { value: "constant", label: "Constant Value" },
+          { value: "today", label: "Current Date" },
+        ],
+        string: [
+          { value: "ffill", label: "Forward Fill" },
+          { value: "most_frequent", label: "Most Frequent" },
+          { value: "constant", label: "Constant Value" },
+        ],
+        text: [
+          { value: "ffill", label: "Forward Fill" },
+          { value: "most_frequent", label: "Most Frequent" },
+          { value: "constant", label: "Constant Value" },
+        ],
+        categorical: [
+          { value: "ffill", label: "Forward Fill" },
+          { value: "categorical", label: "Categorical" },
+          { value: "most_frequent", label: "Most Frequent" },
+          { value: "constant", label: "Constant Value" },
+        ],
+      }.freeze
+
+      def self.constants
+        {
+          preprocessing_strategies: PREPROCESSING_STRATEGIES,
+        }
+      end
+
+      def self.params_by_class
+        @params_by_class ||= {}
+      end
+
+      def self.methods_by_class
+        @methods_by_class ||= {}
+      end
+
+      def self.supported_params
+        @supported_params ||= []
+      end
+
+      def self.supported_methods
+        @supported_methods ||= []
+      end
+
+      def initialize(column)
+        @column = column
+        @dataset = column.dataset
+      end
+
+      class << self
+        def supported_params
+          @supported_params ||= []
+        end
+
+        def supported_methods
+          @supported_methods ||= []
+        end
+      end
+
+      def imputers
+        return {} if column.preprocessing_steps.blank?
+
+        @imputers ||= column.preprocessing_steps.keys.reduce({}) do |hash, key|
+          hash.tap do
+            hash[key.to_sym] = Imputer.new(
+              column,
+              column.preprocessing_steps[key],
+            )
+          end
+        end
+      end
+
+      def training
+        @training ||= imputer_group(:training)
+      end
+
+      def inference
+        @inference ||= imputer_group(:inference)
+      end
+
+      def preprocessing_descriptions
+        return [] if column.preprocessing_steps.blank?
+
+        [training.description].compact
+      end
+
+      private
+
+      def imputer_group(key)
+        imputers.dig(key.to_sym) || NullImputer.new
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/learner.rb

@@ -0,0 +1,18 @@
+module EasyML
+  class Column
+    class Learner
+      attr_accessor :dataset, :column
+
+      def initialize(column)
+        @column = column
+        @dataset = column.dataset
+      end
+
+      def learner
+        @learner ||= EasyML::Column::Learners::Base.adapter(column).new(column)
+      end
+
+      delegate :learn, to: :learner
+    end
+  end
+end