easy_ml 0.2.0.pre.rc57 → 0.2.0.pre.rc60

data/app/models/easy_ml/column/learners/base.rb

@@ -0,0 +1,103 @@
+module EasyML
+  class Column
+    module Learners
+      class Base
+        attr_accessor :column, :dataset, :dtype, :select
+
+        def initialize(column)
+          @column = column
+          @dataset = column.dataset
+          @select = dataset.date_column.present? ? [dataset.date_column.name] : []
+        end
+
+        def self.adapter(column)
+          begin
+            dtype = column.datatype || EasyML::Data::PolarsColumn.determine_type(column.raw.data[column.name])
+          rescue => e
+            raise "Unable to find column #{column.name}. If this column is computed by a feature, you forgot to declare computes_columns"
+          end
+
+          case dtype.to_sym
+          when :float, :integer
+            EasyML::Column::Learners::Numeric
+          when :string, :text
+            EasyML::Column::Learners::String
+          when :categorical
+            EasyML::Column::Learners::Categorical
+          when :datetime, :date
+            EasyML::Column::Learners::Datetime
+          when :boolean
+            EasyML::Column::Learners::Boolean
+          when :null
+            EasyML::Column::Learners::Null
+          else
+            raise "Don't know how to learn from dtype: #{dtype}"
+          end
+        end
+
+        TYPES_ALL = %i(raw clipped processed)
+        TYPES_RAW = %i(raw clipped)
+        TYPES_PROCESSED = %i(processed)
+
+        def types(type = :all)
+          return TYPES_PROCESSED if !column.in_raw_dataset?
+
+          case type
+          when :all then TYPES_ALL
+          when :raw then TYPES_RAW
+          when :processed then TYPES_PROCESSED
+          else
+            TYPES_ALL
+          end
+        end
+
+        def learn(type: :all)
+          types(type).each_with_object({}) do |t, h|
+            h[t] = learn_split(column.send(t))
+          end
+        end
+
+        def full_dataset_statistics(df)
+          return {} if df.nil?
+
+          {
+            num_rows: df.size,
+            null_count: df[column.name].null_count || 0,
+          }
+        end
+
+        def train_statistics(df)
+          return {} if df.nil?
+
+          {
+            last_value: last_value(df),
+            most_frequent_value: df[column.name].mode.sort.to_a&.first,
+          }
+        end
+
+        def learn_split(split)
+          df = split.data(select: select)
+          train_df = split.train(select: select)
+          full_dataset_stats = full_dataset_statistics(df)
+          train_stats = train_statistics(train_df)
+          full_dataset_stats.merge!(train_stats)
+        end
+
+        def last_value(df)
+          return unless dataset.date_column.present?
+          return nil if df.empty? || !df.columns.include?(dataset.date_column.name)
+
+          # Sort by date and get the last non-null value
+          sorted_df = df.sort(dataset.date_column.name, reverse: true)
+          last_value = sorted_df
+            .filter(Polars.col(column.name).is_not_null)
+            .select(column.name)
+            .head(1)
+            .item
+
+          last_value
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/learners/boolean.rb

@@ -0,0 +1,11 @@
+module EasyML
+  class Column
+    module Learners
+      class Boolean < Categorical
+        def sort_by(value)
+          value == true ? 1 : 0
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/learners/categorical.rb

@@ -0,0 +1,51 @@
+module EasyML
+  class Column
+    module Learners
+      class Categorical < String
+        def learn(type)
+          types(type).each_with_object({}) do |type, h|
+            h[type] = case type
+              when :raw then learn_split(column.raw)
+              when :processed then learn_split(column.raw).merge!(null_count: 0)
+              end
+          end
+        end
+
+        def train_statistics(df)
+          return {} if df.nil?
+
+          super(df).merge!({
+            allowed_categories: allowed_categories(df),
+            counts: df[column.name].value_counts.to_hash,
+          }.merge!(learn_encoder_decoder(df)))
+        end
+
+        def learn_encoder_decoder(df)
+          value_counts = df[column.name].value_counts
+          column_names = value_counts.columns
+          value_column = column_names[0]
+          count_column = column_names[1]
+
+          as_hash = value_counts.select([value_column, count_column]).rows.to_a.to_h.transform_keys(&column.method(:cast))
+          label_encoder = as_hash.keys.compact.sort_by(&column.method(:sort_by)).each.with_index.reduce({}) do |h, (k, i)|
+            h.tap do
+              h[k] = i
+            end
+          end
+          label_decoder = label_encoder.invert
+
+          {
+            value: as_hash,
+            label_encoder: label_encoder,
+            label_decoder: label_decoder,
+          }
+        end
+
+        def allowed_categories(df)
+          val_counts = df[column.name].value_counts
+          val_counts[val_counts["count"] >= column.categorical_min][column.name].to_a.compact.sort_by(&column.method(:sort_by))
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/learners/datetime.rb

@@ -0,0 +1,19 @@
+module EasyML
+  class Column
+    module Learners
+      class Datetime < Base
+        def full_dataset_statistics(df)
+          return {} if df.nil?
+
+          super(df).merge!({
+            unique_count: df[column.name].n_unique,
+          })
+        end
+
+        def last_value(df)
+          df[column.name].sort[-1]
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/learners/null.rb

@@ -0,0 +1,22 @@
+module EasyML
+  class Column
+    module Learners
+      class Null < Base
+        def full_dataset_statistics(df)
+          return {} if df.nil?
+
+          {
+            num_rows: df.size,
+            null_count: df[column.name].null_count || 0,
+          }
+        end
+
+        def train_statistics(df)
+          {
+            last_value: last_value(df),
+          }
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/learners/numeric.rb

@@ -0,0 +1,33 @@
+module EasyML
+  class Column
+    module Learners
+      class Numeric < Base
+        def train_statistics(df)
+          return {} if df.nil?
+
+          super(df).merge!({
+            mean: df[column.name].mean,
+            median: df[column.name].median,
+            min: df[column.name].min,
+            max: df[column.name].max,
+            std: df[column.name].std,
+            last_value: last_value(df),
+          }.compact)
+        end
+
+        def last_value(df)
+          if dataset.date_column.present?
+            sorted_df = df.sort(dataset.date_column.name, reverse: true)
+            last_value = sorted_df
+              .filter(Polars.col(column.name).is_not_null)
+              .select(column.name)
+              .head(1)
+              .item
+
+            last_value
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/learners/string.rb

@@ -0,0 +1,15 @@
+module EasyML
+  class Column
+    module Learners
+      class String < Base
+        def full_dataset_statistics(df)
+          return {} if df.nil?
+
+          super(df).merge!({
+            unique_count: df[column.name].cast(:str).n_unique,
+          })
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/lineage/base.rb

@@ -0,0 +1,22 @@
+module EasyML
+  class Column
+    class Lineage
+      class Base
+        attr_accessor :dataset, :column
+
+        def initialize(column)
+          @column = column
+          @dataset = column.dataset
+        end
+
+        def as_json
+          {
+            key: key,
+            description: description,
+            timestamp: timestamp,
+          }.with_indifferent_access
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/lineage/computed_by_feature.rb

@@ -0,0 +1,23 @@
+module EasyML
+  class Column
+    class Lineage
+      class ComputedByFeature < Base
+        def key
+          :computed_by_feature
+        end
+
+        def description
+          "Computed by #{column.computed_by}"
+        end
+
+        def timestamp
+          column.feature.fit_at
+        end
+
+        def check
+          column.computed_by.present?
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/lineage/preprocessed.rb

@@ -0,0 +1,23 @@
+module EasyML
+  class Column
+    class Lineage
+      class Preprocessed < Base
+        def key
+          :preprocessed
+        end
+
+        def description
+          "Preprocessed using #{column.imputers.preprocessing_descriptions.join(", ")}"
+        end
+
+        def timestamp
+          column.dataset.refreshed_at
+        end
+
+        def check
+          column.preprocessing_steps.present?
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/lineage/raw_dataset.rb

@@ -0,0 +1,23 @@
+module EasyML
+  class Column
+    class Lineage
+      class RawDataset < Base
+        def key
+          :raw_dataset
+        end
+
+        def description
+          "Present in raw dataset"
+        end
+
+        def timestamp
+          column.dataset.datasource.refreshed_at
+        end
+
+        def check
+          column.in_raw_dataset?
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/lineage.rb

@@ -0,0 +1,28 @@
+module EasyML
+  class Column
+    class Lineage
+      attr_accessor :dataset, :column
+
+      def initialize(column)
+        @column = column
+        @dataset = column.dataset
+      end
+
+      def sort_order
+        [
+          RawDataset,
+          ComputedByFeature,
+          Preprocessed,
+        ]
+      end
+
+      def lineage
+        sort_order.map do |cl|
+          cl.new(column)
+        end.select(&:check)
+          .map(&:as_json)
+          .compact
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/selector.rb

@@ -0,0 +1,96 @@
+module EasyML
+  class Column
+    class Selector
+      attr_accessor :selected, :dataset, :column, :transform
+
+      def initialize(column, selected = nil, &block)
+        @column = column
+        @dataset = column.dataset
+        @selected = selected
+        @transform = block
+      end
+
+      def name
+        column.name
+      end
+
+      def raw
+        if column.is_computed? && !column.in_raw_dataset?
+          Selector.new(column, :processed)
+        else
+          Selector.new(column, :raw)
+        end
+      end
+
+      def clipped
+        Selector.new(column, :raw) do |df|
+          column.imputers.training.clip(df)
+        end
+      end
+
+      def processed
+        Selector.new(column, :processed)
+      end
+
+      def train(**kwargs)
+        select(:train, **kwargs)
+      end
+
+      def test(**kwargs)
+        select(:test, **kwargs)
+      end
+
+      def valid(**kwargs)
+        select(:valid, **kwargs)
+      end
+
+      def data(**kwargs)
+        if column.is_computed?
+          Selector.new(column, :processed).send(:select, :data, **kwargs)
+        else
+          select(:data, **kwargs)
+        end
+      end
+
+      private
+
+      def select(segment, **orig_kwargs)
+        kwargs = orig_kwargs.clone
+        return nil if dataset.nil?
+
+        kwargs[:all_columns] = true
+
+        if kwargs.key?(:select)
+          kwargs[:select] = [kwargs[:select]].flatten
+        else
+          kwargs[:select] = []
+        end
+
+        if (selected == :processed || (selected.nil? && !dataset.needs_refresh?)) && column.one_hot?
+          kwargs[:select] << column.virtual_columns
+        else
+          kwargs[:select] << column.name
+        end
+        kwargs[:select] = kwargs[:select].flatten.uniq
+
+        if @selected.present?
+          available_columns = dataset.send(@selected).send(segment, limit: 1, all_columns: true)&.columns || []
+          kwargs[:select] = available_columns & kwargs[:select]
+          return Polars::DataFrame.new if kwargs[:select].empty?
+          result = dataset.send(@selected).send(segment, **kwargs)
+        else
+          available_columns = dataset.send(segment, limit: 1, all_columns: true)&.columns || []
+          kwargs[:select] = available_columns & kwargs[:select]
+          return Polars::DataFrame.new if kwargs[:select].empty?
+          result = dataset.send(segment, **kwargs)
+        end
+
+        if transform
+          result = transform.call(result)
+        end
+
+        result
+      end
+    end
+  end
+end