easy_ml 0.2.0.pre.rc58 → 0.2.0.pre.rc61

data/app/models/easy_ml/column/imputers/ffill.rb

@@ -0,0 +1,29 @@
+module EasyML
+  class Column
+    class Imputers
+      class Ffill < Base
+        method_applies :ffill
+
+        def self.description
+          "Forward fill imputation"
+        end
+
+        def transform(df)
+          return df unless last_value.present?
+
+          df = df.with_column(
+            Polars.when(Polars.col(column.name).is_null)
+                  .then(Polars.lit(last_value).cast(column.polars_datatype))
+                  .otherwise(Polars.col(column.name).cast(column.polars_datatype))
+                  .alias(column.name)
+          )
+          df
+        end
+
+        def last_value
+          statistics(:last_value)
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/imputers/imputer.rb

@@ -0,0 +1,103 @@
+module EasyML
+  class Column
+    class Imputers
+      class Imputer
+        attr_accessor :dataset, :column, :preprocessing_step
+
+        def initialize(column, preprocessing_step)
+          @column = column
+          @dataset = column.dataset
+          @preprocessing_step = preprocessing_step.with_indifferent_access
+          validate_preprocessing_step!
+        end
+
+        def inspect
+          "#<#{self.class.name} adapters=#{adapters.map(&:inspect).join(", ")}>"
+        end
+
+        def ordered_adapters
+          [
+            Clip,
+            Mean,
+            Median,
+            Constant,
+            Ffill,
+            Categorical,
+            MostFrequent,
+            Today,
+            OneHotEncoder,
+            OrdinalEncoder,
+          ]
+        end
+
+        def adapters
+          @adapters ||= ordered_adapters.map { |klass| klass.new(column, preprocessing_step) }.select(&:applies?)
+        end
+
+        def imputers
+          return nil if column.preprocessing_steps.blank?
+
+          @imputers ||= column.preprocessing_steps.keys.reduce({}) do |hash, key|
+            hash[key.to_sym] = Imputer.new(
+              column: column,
+              preprocessing_step: column.preprocessing_steps[key],
+            )
+          end
+        end
+
+        def description
+          adapters.map(&:description).compact.join(", ")
+        end
+
+        def anything?
+          adapters.any?
+        end
+
+        def transform(df)
+          return df unless anything?
+
+          adapters.reduce(df) do |df, adapter|
+            adapter.transform(df)
+          end
+        end
+
+        def clip(df)
+          return df unless adapters.map(&:class).include?(Clip)
+
+          EasyML::Column::Imputers::Clip.new(column, preprocessing_step).transform(df)
+        end
+
+        def decode_labels(df)
+          return df unless adapters.map(&:class).include?(OrdinalEncoder)
+
+          EasyML::Column::Imputers::OrdinalEncoder.new(column, preprocessing_step).decode_labels(df)
+        end
+
+        private
+
+        def validate_preprocessing_step!
+          validate_params!
+          validate_method!
+        end
+
+        def validate_params!
+          return unless preprocessing_step[:params]
+
+          preprocessing_step[:params].keys.each do |param|
+            unless Imputers.supported_params.include?(param.to_sym)
+              raise ArgumentError, "Unsupported preprocessing parameter '#{param}'. Supported parameters are: #{Imputers.supported_params.join(", ")}"
+            end
+          end
+        end
+
+        def validate_method!
+          return unless preprocessing_step[:method]
+
+          unless Imputers.supported_methods.include?(preprocessing_step[:method].to_sym)
+            raise ArgumentError, "Unsupported preprocessing method '#{preprocessing_step[:method]}'. Supported methods are: #{Imputers.supported_methods.join(", ")}"
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/imputers/mean.rb

@@ -0,0 +1,27 @@
+module EasyML
+  class Column
+    class Imputers
+      class Mean < Base
+        method_applies :mean
+
+        def self.description
+          "Mean imputation"
+        end
+
+        def transform(df)
+          return df unless mean.present?
+
+          mean = statistics(:mean)
+          df = df.with_column(
+            Polars.col(column.name).fill_null(mean).alias(column.name)
+          )
+          df
+        end
+
+        def mean
+          statistics(:mean)
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/imputers/median.rb

@@ -0,0 +1,27 @@
+module EasyML
+  class Column
+    class Imputers
+      class Median < Base
+        method_applies :median
+
+        def self.description
+          "Median imputation"
+        end
+
+        def transform(df)
+          return df unless median.present?
+
+          median = statistics(:median)
+          df = df.with_column(
+            Polars.col(column.name).fill_null(median).alias(column.name)
+          )
+          df
+        end
+
+        def median
+          statistics(:median)
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/imputers/most_frequent.rb

@@ -0,0 +1,27 @@
+module EasyML
+  class Column
+    class Imputers
+      class MostFrequent < Base
+        method_applies :most_frequent
+
+        def self.description
+          "Most frequent value imputation"
+        end
+
+        def transform(df)
+          return df unless most_frequent.present?
+
+          most_frequent = statistics(:most_frequent_value)
+          df = df.with_column(
+            Polars.col(column.name).fill_null(most_frequent).alias(column.name)
+          )
+          df
+        end
+
+        def most_frequent
+          statistics(:most_frequent_value)
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/imputers/null_imputer.rb

@@ -0,0 +1,15 @@
+module EasyML
+  class Column
+    class Imputers
+      class NullImputer
+        def anything?
+          false
+        end
+
+        def method_missing(_name, df)
+          df
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/imputers/one_hot_encoder.rb

@@ -0,0 +1,30 @@
+module EasyML
+  class Column
+    class Imputers
+      class OneHotEncoder < Base
+        param_applies :one_hot
+
+        def self.description
+          "One-hot encoder"
+        end
+
+        def transform(df)
+          return df unless allowed_categories.present?
+
+          allowed_categories.each do |value|
+            new_col_name = "#{column.name}_#{value}".gsub(/-/, "_")
+            df = df.with_column(
+              df[column.name].cast(Polars::String).eq(value.to_s).cast(Polars::Boolean).alias(new_col_name)
+            )
+          end
+          df = df.drop([column.name])
+          df
+        end
+
+        def allowed_categories
+          column.allowed_categories.sort
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/imputers/ordinal_encoder.rb

@@ -0,0 +1,78 @@
+module EasyML
+  class Column
+    class Imputers
+      class OrdinalEncoder < Base
+        param_applies :ordinal_encoding
+
+        def self.description
+          "Ordinal encoder"
+        end
+
+        def transform(df)
+          return df unless label_encoder.present?
+
+          case column.datatype
+          when :categorical
+            df = df.with_column(
+              Polars.when(Polars.col(column.name).is_in(allowed_categories))
+                .then(Polars.col(column.name))
+                .otherwise(Polars.lit("other"))
+                .alias(column.name)
+            )
+          when :boolean
+            # no-op
+          end
+
+          df = df.with_column(
+            df[column.name].map { |v| label_encoder[column.cast(v)] || other_value }.alias(column.name)
+          )
+
+          df
+        end
+
+        def decode_labels(df)
+          if df.is_a?(Array)
+            return df.map { |v| label_decoder[v.to_i] }
+          end
+
+          df = df.with_column(
+            df[column.name].map { |v| label_decoder[v.to_i] }.alias(column.name)
+          )
+          df
+        end
+
+        def categories
+          label_encoder.keys
+        end
+
+        def values
+          label_encoder.values
+        end
+
+        def cast_encoder(encoder)
+          encoder.transform_keys { |k| column.cast(k) }
+        end
+
+        def cast_decoder(decoder)
+          decoder.transform_keys { |k| k.to_i }
+        end
+
+        def label_encoder
+          @label_encoder ||= cast_encoder(statistics(:label_encoder))
+        end
+
+        def label_decoder
+          @label_decoder ||= cast_decoder(statistics(:label_decoder))
+        end
+
+        def other_value
+          label_encoder.values.max + 1
+        end
+
+        def allowed_categories
+          column.allowed_categories
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/imputers/today.rb

@@ -0,0 +1,20 @@
+module EasyML
+  class Column
+    class Imputers
+      class Today < Base
+        method_applies :today
+
+        def self.description
+          "Current date imputation"
+        end
+
+        def transform(df)
+          df = df.with_column(
+            Polars.col(column.name).fill_null(Polars.lit(UTC.today.beginning_of_day)).alias(column.name)
+          )
+          df
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/imputers.rb

@@ -0,0 +1,126 @@
+module EasyML
+  class Column
+    class Imputers
+      attr_accessor :dataset, :column
+
+      ALLOWED_PARAMS = {
+        constant: [:constant],
+        categorical: %i[categorical_min one_hot ordinal_encoding],
+        most_frequent: %i[one_hot ordinal_encoding],
+        mean: [:clip],
+        median: [:clip],
+      }
+
+      PREPROCESSING_STRATEGIES = {
+        float: [
+          { value: "ffill", label: "Forward Fill" },
+          { value: "mean", label: "Mean" },
+          { value: "median", label: "Median" },
+          { value: "constant", label: "Constant Value" },
+        ],
+        integer: [
+          { value: "ffill", label: "Forward Fill" },
+          { value: "mean", label: "Mean" },
+          { value: "median", label: "Median" },
+          { value: "constant", label: "Constant Value" },
+        ],
+        boolean: [
+          { value: "ffill", label: "Forward Fill" },
+          { value: "most_frequent", label: "Most Frequent" },
+          { value: "constant", label: "Constant Value" },
+        ],
+        datetime: [
+          { value: "ffill", label: "Forward Fill" },
+          { value: "constant", label: "Constant Value" },
+          { value: "today", label: "Current Date" },
+        ],
+        string: [
+          { value: "ffill", label: "Forward Fill" },
+          { value: "most_frequent", label: "Most Frequent" },
+          { value: "constant", label: "Constant Value" },
+        ],
+        text: [
+          { value: "ffill", label: "Forward Fill" },
+          { value: "most_frequent", label: "Most Frequent" },
+          { value: "constant", label: "Constant Value" },
+        ],
+        categorical: [
+          { value: "ffill", label: "Forward Fill" },
+          { value: "categorical", label: "Categorical" },
+          { value: "most_frequent", label: "Most Frequent" },
+          { value: "constant", label: "Constant Value" },
+        ],
+      }.freeze
+
+      def self.constants
+        {
+          preprocessing_strategies: PREPROCESSING_STRATEGIES,
+        }
+      end
+
+      def self.params_by_class
+        @params_by_class ||= {}
+      end
+
+      def self.methods_by_class
+        @methods_by_class ||= {}
+      end
+
+      def self.supported_params
+        @supported_params ||= []
+      end
+
+      def self.supported_methods
+        @supported_methods ||= []
+      end
+
+      def initialize(column)
+        @column = column
+        @dataset = column.dataset
+      end
+
+      class << self
+        def supported_params
+          @supported_params ||= []
+        end
+
+        def supported_methods
+          @supported_methods ||= []
+        end
+      end
+
+      def imputers
+        return {} if column.preprocessing_steps.blank?
+
+        @imputers ||= column.preprocessing_steps.keys.reduce({}) do |hash, key|
+          hash.tap do
+            hash[key.to_sym] = Imputer.new(
+              column,
+              column.preprocessing_steps[key],
+            )
+          end
+        end
+      end
+
+      def training
+        @training ||= imputer_group(:training)
+      end
+
+      def inference
+        @inference ||= imputer_group(:inference)
+      end
+
+      def preprocessing_descriptions
+        return [] if column.preprocessing_steps.blank?
+
+        [training.description].compact
+      end
+
+      private
+
+      def imputer_group(key)
+        imputers.dig(key.to_sym) || NullImputer.new
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/learner.rb

@@ -0,0 +1,18 @@
+module EasyML
+  class Column
+    class Learner
+      attr_accessor :dataset, :column
+
+      def initialize(column)
+        @column = column
+        @dataset = column.dataset
+      end
+
+      def learner
+        @learner ||= EasyML::Column::Learners::Base.adapter(column).new(column)
+      end
+
+      delegate :learn, to: :learner
+    end
+  end
+end

data/app/models/easy_ml/column/learners/base.rb

@@ -0,0 +1,103 @@
+module EasyML
+  class Column
+    module Learners
+      class Base
+        attr_accessor :column, :dataset, :dtype, :select
+
+        def initialize(column)
+          @column = column
+          @dataset = column.dataset
+          @select = dataset.date_column.present? ? [dataset.date_column.name] : []
+        end
+
+        def self.adapter(column)
+          begin
+            dtype = column.datatype || EasyML::Data::PolarsColumn.determine_type(column.raw.data[column.name])
+          rescue => e
+            raise "Unable to find column #{column.name}. If this column is computed by a feature, you forgot to declare computes_columns"
+          end
+
+          case dtype.to_sym
+          when :float, :integer
+            EasyML::Column::Learners::Numeric
+          when :string, :text
+            EasyML::Column::Learners::String
+          when :categorical
+            EasyML::Column::Learners::Categorical
+          when :datetime, :date
+            EasyML::Column::Learners::Datetime
+          when :boolean
+            EasyML::Column::Learners::Boolean
+          when :null
+            EasyML::Column::Learners::Null
+          else
+            raise "Don't know how to learn from dtype: #{dtype}"
+          end
+        end
+
+        TYPES_ALL = %i(raw clipped processed)
+        TYPES_RAW = %i(raw clipped)
+        TYPES_PROCESSED = %i(processed)
+
+        def types(type = :all)
+          return TYPES_PROCESSED if !column.in_raw_dataset?
+
+          case type
+          when :all then TYPES_ALL
+          when :raw then TYPES_RAW
+          when :processed then TYPES_PROCESSED
+          else
+            TYPES_ALL
+          end
+        end
+
+        def learn(type: :all)
+          types(type).each_with_object({}) do |t, h|
+            h[t] = learn_split(column.send(t))
+          end
+        end
+
+        def full_dataset_statistics(df)
+          return {} if df.nil?
+
+          {
+            num_rows: df.size,
+            null_count: df[column.name].null_count || 0,
+          }
+        end
+
+        def train_statistics(df)
+          return {} if df.nil?
+
+          {
+            last_value: last_value(df),
+            most_frequent_value: df[column.name].mode.sort.to_a&.first,
+          }
+        end
+
+        def learn_split(split)
+          df = split.data(select: select)
+          train_df = split.train(select: select)
+          full_dataset_stats = full_dataset_statistics(df)
+          train_stats = train_statistics(train_df)
+          full_dataset_stats.merge!(train_stats)
+        end
+
+        def last_value(df)
+          return unless dataset.date_column.present?
+          return nil if df.empty? || !df.columns.include?(dataset.date_column.name)
+
+          # Sort by date and get the last non-null value
+          sorted_df = df.sort(dataset.date_column.name, reverse: true)
+          last_value = sorted_df
+            .filter(Polars.col(column.name).is_not_null)
+            .select(column.name)
+            .head(1)
+            .item
+
+          last_value
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/learners/boolean.rb

@@ -0,0 +1,11 @@
+module EasyML
+  class Column
+    module Learners
+      class Boolean < Categorical
+        def sort_by(value)
+          value == true ? 1 : 0
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/learners/categorical.rb

@@ -0,0 +1,51 @@
+module EasyML
+  class Column
+    module Learners
+      class Categorical < String
+        def learn(type)
+          types(type).each_with_object({}) do |type, h|
+            h[type] = case type
+              when :raw then learn_split(column.raw)
+              when :processed then learn_split(column.raw).merge!(null_count: 0)
+              end
+          end
+        end
+
+        def train_statistics(df)
+          return {} if df.nil?
+
+          super(df).merge!({
+            allowed_categories: allowed_categories(df),
+            counts: df[column.name].value_counts.to_hash,
+          }.merge!(learn_encoder_decoder(df)))
+        end
+
+        def learn_encoder_decoder(df)
+          value_counts = df[column.name].value_counts
+          column_names = value_counts.columns
+          value_column = column_names[0]
+          count_column = column_names[1]
+
+          as_hash = value_counts.select([value_column, count_column]).rows.to_a.to_h.transform_keys(&column.method(:cast))
+          label_encoder = as_hash.keys.compact.sort_by(&column.method(:sort_by)).each.with_index.reduce({}) do |h, (k, i)|
+            h.tap do
+              h[k] = i
+            end
+          end
+          label_decoder = label_encoder.invert
+
+          {
+            value: as_hash,
+            label_encoder: label_encoder,
+            label_decoder: label_decoder,
+          }
+        end
+
+        def allowed_categories(df)
+          val_counts = df[column.name].value_counts
+          val_counts[val_counts["count"] >= column.categorical_min][column.name].to_a.compact.sort_by(&column.method(:sort_by))
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/learners/datetime.rb

@@ -0,0 +1,19 @@
+module EasyML
+  class Column
+    module Learners
+      class Datetime < Base
+        def full_dataset_statistics(df)
+          return {} if df.nil?
+
+          super(df).merge!({
+            unique_count: df[column.name].n_unique,
+          })
+        end
+
+        def last_value(df)
+          df[column.name].sort[-1]
+        end
+      end
+    end
+  end
+end