easy_ml 0.2.0.pre.rc71 → 0.2.0.pre.rc75

data/app/models/easy_ml/column_list.rb

@@ -1,6 +1,7 @@
 module EasyML
   module ColumnList
     include Historiographer::Relation
+    include EasyML::Timing
 
     def sync(delete: true)
       return unless dataset.schema.present?
@@ -39,35 +40,28 @@ module EasyML
       df
     end
 
+    measure_method_timing :transform
+
+    def apply_clip(df)
+      clip_cols = has_clip.raw
+      return df unless clip_cols.any?
+
+      clipped_exprs = EasyML::ColumnList::Imputer.new(
+        dataset,
+        df,
+        columns: clip_cols,
+        imputers: [:clip],
+      ).exprs
+
+      df.with_columns(clipped_exprs)
+    end
+
     def learn(type: :raw, computed: false)
-      cols_to_learn = column_list.reload.needs_learn
-      cols_to_learn = cols_to_learn.computed if computed
-      cols_to_learn = cols_to_learn.select(&:persisted?).reject(&:empty?)
-      cols_to_learn.each { |col| col.learn(type: type) }
-      EasyML::Column.import(cols_to_learn, on_duplicate_key_update: { columns: %i[
-                                             statistics
-                                             learned_at
-                                             sample_values
-                                             last_datasource_sha
-                                             is_learning
-                                             datatype
-                                             polars_datatype
-                                           ] })
-      set_feature_lineage
+      EasyML::Dataset::Learner.new(dataset, type: type).learn
       reload
     end
 
-    def set_feature_lineage
-      names = dataset.features.computed_column_names
-      columns = where(name: names, computed_by: nil).map do |col|
-        col.assign_attributes(
-          is_computed: true,
-          computed_by: col.computing_feature&.name,
-        )
-        col
-      end
-      EasyML::Column.import(columns, on_duplicate_key_update: { columns: %i[ is_computed computed_by ] })
-    end
+    measure_method_timing :learn
 
     def statistics
       stats = { raw: {}, processed: {} }
@@ -115,6 +109,25 @@ module EasyML
       column_list.sort_by { |col| [col.sort_required, col.name] }
     end
 
+    def set_feature_lineage(cols_to_learn)
+      names = dataset.features.computed_column_names
+      columns = where(name: names, computed_by: nil).map do |col|
+        col.assign_attributes(
+          is_computed: true,
+          computed_by: col.computing_feature&.name,
+        )
+        col
+      end
+      EasyML::Column.import(columns, on_duplicate_key_update: { columns: %i[ is_computed computed_by ] })
+
+      lineage = cols_to_learn.flat_map do |col|
+        EasyML::Lineage.learn(col)
+      end.compact
+      EasyML::Lineage.import(lineage, on_duplicate_key_update: { columns: %i[ column_id key occurred_at description ] })
+    end
+
+    measure_method_timing :set_feature_lineage
+
     private
 
     def import_new(new_columns, existing_columns)
@@ -127,7 +140,7 @@ module EasyML
         col
       end
       EasyML::Column.import(cols_to_insert)
-      set_feature_lineage
+      set_feature_lineage(cols_to_insert)
       column_list.reload
     end
 

data/app/models/easy_ml/dataset/learner/base.rb

@@ -0,0 +1,34 @@
+module EasyML
+  class Dataset
+    class Learner
+      class Base
+        attr_reader :dataset, :columns, :type
+
+        def initialize(dataset, columns, type: :raw)
+          @dataset = dataset
+          @columns = columns
+          @type = type
+        end
+
+        def skip_processing?(column, type)
+          (!column.in_raw_dataset? && type.to_sym != :processed) ||
+            (column.one_hot? && type.to_sym == :processed)
+        end
+
+        TYPES_ALL = %i(raw clipped processed)
+        TYPES_RAW = %i(raw clipped)
+        TYPES_PROCESSED = %i(processed)
+
+        def types(type = :all)
+          case type
+          when :all then TYPES_ALL
+          when :raw then TYPES_RAW
+          when :processed then TYPES_PROCESSED
+          else
+            TYPES_ALL
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/dataset/learner/eager/boolean.rb

@@ -0,0 +1,10 @@
+module EasyML
+  class Dataset
+    class Learner
+      class Eager
+        class Boolean < Categorical
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/dataset/learner/eager/categorical.rb

@@ -0,0 +1,51 @@
+module EasyML
+  class Dataset
+    class Learner
+      class Eager
+        class Categorical < Query
+          def train_query(df)
+            {
+              counts: counts(df).to_hash,
+              allowed_categories: allowed_categories(df).to_series.to_a,
+            }.merge!(
+              learn_encoder_decoder(df)
+            )
+          end
+
+          def learn_encoder_decoder(df)
+            unsorted = allowed_categories(df).lazy.with_row_count.collect.to_hash.invert
+
+            label_encoder = unsorted.transform_keys(&column.method(:cast)).keys.compact.sort_by(&column.method(:sort_by)).each.with_index.reduce({}) do |h, (k, i)|
+              h.tap do
+                h[k] = i
+              end
+            end
+            label_decoder = label_encoder.invert
+
+            {
+              label_encoder: label_encoder,
+              label_decoder: label_decoder,
+            }
+          end
+
+          def counts(df)
+            return @counts if @counts
+
+            @counts = df.group_by(column.name)
+                        .agg(Polars.col(column.name).count.alias("count"))
+          end
+
+          def allowed_categories(df)
+            return @allowed_categories if @allowed_categories
+
+            @allowed_categories = df.join(counts(df), on: column.name)
+              .filter(Polars.col("count").ge(column.categorical_min))
+              .select(column.name)
+              .unique
+              .sort(column.name, reverse: true)
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/dataset/learner/eager/query.rb

@@ -0,0 +1,37 @@
+module EasyML
+  class Dataset
+    class Learner
+      class Eager
+        class Query < EasyML::Dataset::Learner::Query
+          def execute(split, df)
+            case split.to_sym
+            when :train
+              train_query(df)
+            when :data
+              full_dataset_query(df)
+            end
+          end
+
+          def train_query(df)
+            {}
+          end
+
+          def full_dataset_query(df)
+            {}
+          end
+
+          def adapter
+            case (raw_dtype&.to_sym || dtype.to_sym)
+            when :categorical
+              Eager::Categorical
+            when :boolean
+              Eager::Boolean
+            else
+              nil
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/dataset/learner/eager.rb

@@ -0,0 +1,43 @@
+module EasyML
+  class Dataset
+    class Learner
+      class Eager < Base
+        def learn
+          types.reduce({}) do |h, type|
+            h.tap do
+              h[type] = learn_using_split(:train, type).deep_merge!(learn_using_split(:data, type))
+            end
+          end
+        end
+
+        private
+
+        def learn_using_split(split, type)
+          return {} if @dataset.send(type).empty?
+
+          execute_queries(split, type) || {}
+        end
+
+        def fetch_df(split, type)
+          @dataset.send(type).send(split, all_columns: true)
+        end
+
+        def execute_queries(split, type)
+          @fetched = nil
+
+          columns.reduce({}) do |h, column|
+            h.tap do
+              next if skip_processing?(column, type)
+
+              adapter = Eager::Query.new(@dataset, column).adapter
+              next unless adapter.present?
+
+              @fetched ||= fetch_df(split, type)
+              h[column.name] = adapter.new(@dataset, column).execute(split, @fetched)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/dataset/learner/lazy/boolean.rb

@@ -0,0 +1,13 @@
+module EasyML
+  class Dataset
+    class Learner
+      class Lazy
+        class Boolean < Categorical
+          def sort_by(value)
+            value == true ? 1 : 0
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/dataset/learner/lazy/categorical.rb

@@ -0,0 +1,10 @@
+module EasyML
+  class Dataset
+    class Learner
+      class Lazy
+        class Categorical < String
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/dataset/learner/lazy/datetime.rb

@@ -0,0 +1,19 @@
+module EasyML
+  class Dataset
+    class Learner
+      class Lazy
+        class Datetime < Query
+          def full_dataset_query
+            super.concat([
+              unique_count,
+            ])
+          end
+
+          def unique_count
+            Polars.col(column.name).n_unique.alias("#{column.name}__unique_count")
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/dataset/learner/lazy/null.rb

@@ -0,0 +1,17 @@
+module EasyML
+  class Dataset
+    class Learner
+      class Lazy
+        class Null < Query
+          def full_dataset_query
+            []
+          end
+
+          def train_query
+            []
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/dataset/learner/lazy/numeric.rb

@@ -0,0 +1,19 @@
+module EasyML
+  class Dataset
+    class Learner
+      class Lazy
+        class Numeric < Query
+          def train_query
+            super.concat([
+              Polars.col(column.name).mean.alias("#{column.name}__mean"),
+              Polars.col(column.name).median.alias("#{column.name}__median"),
+              Polars.col(column.name).min.alias("#{column.name}__min"),
+              Polars.col(column.name).max.alias("#{column.name}__max"),
+              Polars.col(column.name).std.alias("#{column.name}__std"),
+            ])
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/dataset/learner/lazy/query.rb

@@ -0,0 +1,69 @@
+module EasyML
+  class Dataset
+    class Learner
+      class Lazy
+        class Query < EasyML::Dataset::Learner::Query
+          def adapter
+            case dtype.to_sym
+            when :float, :integer
+              Lazy::Numeric
+            when :string, :text
+              Lazy::String
+            when :categorical
+              Lazy::Categorical
+            when :datetime, :date
+              Lazy::Datetime
+            when :boolean
+              Lazy::Boolean
+            when :null
+              Lazy::Null
+            else
+              raise "Don't know how to learn from dtype: #{dtype}"
+            end
+          end
+
+          def execute(split)
+            case split.to_sym
+            when :train
+              train_query
+            when :data
+              full_dataset_query
+            end
+          end
+
+          private
+
+          def full_dataset_query
+            [num_rows, null_count].compact
+          end
+
+          def train_query
+            [last_value, most_frequent_value].compact
+          end
+
+          def null_count
+            Polars.col(column.name).null_count.alias("#{column.name}__null_count")
+          end
+
+          def num_rows
+            Polars.col(column.name).len.alias("#{column.name}__num_rows")
+          end
+
+          def most_frequent_value
+            Polars.col(column.name).filter(Polars.col(column.name).is_not_null).mode.first.alias("#{column.name}__most_frequent_value")
+          end
+
+          def last_value
+            return unless dataset.date_column.present?
+
+            Polars.col(column.name)
+              .sort_by(dataset.date_column.name, reverse: true, nulls_last: true)
+              .filter(Polars.col(column.name).is_not_null)
+              .first
+              .alias("#{column.name}__last_value")
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/dataset/learner/lazy/string.rb

@@ -0,0 +1,19 @@
+module EasyML
+  class Dataset
+    class Learner
+      class Lazy
+        class String < Query
+          def full_dataset_query
+            super.concat([
+              unique_count,
+            ])
+          end
+
+          def unique_count
+            Polars.col(column.name).cast(:str).n_unique.alias("#{column.name}__unique_count")
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/dataset/learner/lazy.rb

@@ -0,0 +1,51 @@
+module EasyML
+  class Dataset
+    class Learner
+      class Lazy < Base
+        def learn
+          # types.map
+          types.reduce({}) do |h, type|
+            h.tap do
+              h[type] = learn_using_split(:train, type).deep_merge!(learn_using_split(:data, type))
+            end
+          end
+        end
+
+        private
+
+        def learn_using_split(split, type)
+          return {} if @dataset.send(type).empty?
+
+          get_column_statistics(run_queries(split, type))
+        end
+
+        def run_queries(split, type)
+          queries = build_queries(split, type)
+          @dataset.send(type).send(split, all_columns: true, lazy: true).select(queries).collect
+        end
+
+        def get_column_statistics(query_results)
+          query_results.columns.group_by { |k| k.split("__").first }.reduce({}) do |h, (k, v)|
+            h.tap do
+              h[k] ||= {}
+              v.each do |col|
+                statistic_name = col.split("__").last
+                h[k][statistic_name] = query_results[col][0]
+              end
+            end
+          end
+        end
+
+        def build_queries(split, type)
+          columns.flat_map do |column|
+            next if skip_processing?(column, type)
+
+            query = Lazy::Query.new(@dataset, column)
+            query_adapter = query.adapter.new(@dataset, column)
+            query_adapter.execute(split)
+          end.compact
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/dataset/learner/query.rb

@@ -0,0 +1,25 @@
+module EasyML
+  class Dataset
+    class Learner
+      class Query
+        attr_reader :dataset, :column, :dtype, :raw_dtype
+
+        def initialize(dataset, column)
+          @dataset = dataset
+          @column = column
+          begin
+            # TODO: LAZIFY THIS
+            @dtype = column.datatype || EasyML::Data::PolarsColumn.determine_type(column.raw.data[column.name])
+            @raw_dtype = column.raw_dtype
+          rescue => e
+            raise "Unable to find column #{column.name}. If this column is computed by a feature, you forgot to declare computes_columns"
+          end
+        end
+
+        def execute(split)
+          adapter.execute(split)
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/dataset/learner.rb

@@ -0,0 +1,100 @@
+module EasyML
+  class Dataset
+    class Learner
+      include EasyML::Timing
+      attr_accessor :dataset, :columns, :type, :computed, :raw_columns, :statistics
+
+      def initialize(dataset, type: :raw)
+        @dataset = dataset
+        @columns = dataset.columns.reload.needs_learn.sort_by(&:name)
+
+        if computed
+          @columns = @columns.computed
+        end
+
+        @columns = @columns.select(&:persisted?).reject(&:empty?)
+        @type = type
+      end
+
+      def learn
+        prepare
+        learn_statistics
+        save_statistics
+      end
+
+      private
+
+      def save_statistics
+        columns.each do |col|
+          col.merge_statistics(statistics.dig(col.name))
+          col.set_sample_values
+          col.assign_attributes(
+            learned_at: EasyML::Support::UTC.now,
+            last_datasource_sha: col.dataset.last_datasource_sha,
+            last_feature_sha: col.feature&.sha,
+            is_learning: type == :raw,
+          )
+        end
+
+        EasyML::Column.import(columns, on_duplicate_key_update: { columns: %i[
+                                         statistics
+                                         learned_at
+                                         sample_values
+                                         last_datasource_sha
+                                         is_learning
+                                       ] })
+        dataset.columns.set_feature_lineage(columns)
+      end
+
+      measure_method_timing :save_statistics
+
+      def learn_statistics
+        return @statistics if @statistics
+
+        @statistics = lazy_statistics.deep_merge!(eager_statistics).reduce({}) do |h, (type, stat_group)|
+          h.tap do
+            stat_group.each do |statistic, value|
+              h[statistic] ||= {}
+              h[statistic][type] = value
+            end
+          end
+        end.with_indifferent_access
+
+        if type != :raw
+          columns.select(&:one_hot?).each do |column|
+            @statistics[column.name][:processed] = @statistics[column.name][:raw]
+          end
+        end
+      end
+
+      measure_method_timing :learn_statistics
+
+      def prepare
+        @schema = EasyML::Data::PolarsSchema.simplify(@dataset.raw_schema).symbolize_keys
+        @raw_columns = @schema.keys.sort.map(&:to_s)
+        columns.each do |column|
+          attrs = {
+            in_raw_dataset: @raw_columns.include?(column.name),
+            datatype: column.read_attribute(:datatype).present? ? nil : @schema[column.name.to_sym],
+          }.compact
+          column.assign_attributes(attrs)
+        end
+        EasyML::Column.import(columns, on_duplicate_key_update: { columns: %i[in_raw_dataset datatype] })
+      end
+
+      measure_method_timing :prepare
+
+      def lazy_statistics
+        Lazy.new(dataset, columns, type: type).learn
+      end
+
+      measure_method_timing :lazy_statistics
+
+      def eager_statistics
+        Eager.new(dataset, columns, type: type).learn
+      end
+
+      measure_method_timing :eager_statistics
+    end
+  end
+end