easy_ml 0.2.0.pre.rc72 → 0.2.0.pre.rc75

data/app/models/easy_ml/dataset/learner/eager/query.rb

@@ -0,0 +1,37 @@
+module EasyML
+  class Dataset
+    class Learner
+      class Eager
+        class Query < EasyML::Dataset::Learner::Query
+          def execute(split, df)
+            case split.to_sym
+            when :train
+              train_query(df)
+            when :data
+              full_dataset_query(df)
+            end
+          end
+
+          def train_query(df)
+            {}
+          end
+
+          def full_dataset_query(df)
+            {}
+          end
+
+          def adapter
+            case (raw_dtype&.to_sym || dtype.to_sym)
+            when :categorical
+              Eager::Categorical
+            when :boolean
+              Eager::Boolean
+            else
+              nil
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/dataset/learner/eager.rb

@@ -0,0 +1,43 @@
+module EasyML
+  class Dataset
+    class Learner
+      class Eager < Base
+        def learn
+          types.reduce({}) do |h, type|
+            h.tap do
+              h[type] = learn_using_split(:train, type).deep_merge!(learn_using_split(:data, type))
+            end
+          end
+        end
+
+        private
+
+        def learn_using_split(split, type)
+          return {} if @dataset.send(type).empty?
+
+          execute_queries(split, type) || {}
+        end
+
+        def fetch_df(split, type)
+          @dataset.send(type).send(split, all_columns: true)
+        end
+
+        def execute_queries(split, type)
+          @fetched = nil
+
+          columns.reduce({}) do |h, column|
+            h.tap do
+              next if skip_processing?(column, type)
+
+              adapter = Eager::Query.new(@dataset, column).adapter
+              next unless adapter.present?
+
+              @fetched ||= fetch_df(split, type)
+              h[column.name] = adapter.new(@dataset, column).execute(split, @fetched)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/dataset/learner/lazy/boolean.rb

@@ -0,0 +1,13 @@
+module EasyML
+  class Dataset
+    class Learner
+      class Lazy
+        class Boolean < Categorical
+          def sort_by(value)
+            value == true ? 1 : 0
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/dataset/learner/lazy/categorical.rb

@@ -0,0 +1,10 @@
+module EasyML
+  class Dataset
+    class Learner
+      class Lazy
+        class Categorical < String
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/dataset/learner/lazy/datetime.rb

@@ -0,0 +1,19 @@
+module EasyML
+  class Dataset
+    class Learner
+      class Lazy
+        class Datetime < Query
+          def full_dataset_query
+            super.concat([
+              unique_count,
+            ])
+          end
+
+          def unique_count
+            Polars.col(column.name).n_unique.alias("#{column.name}__unique_count")
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/dataset/learner/lazy/null.rb

@@ -0,0 +1,17 @@
+module EasyML
+  class Dataset
+    class Learner
+      class Lazy
+        class Null < Query
+          def full_dataset_query
+            []
+          end
+
+          def train_query
+            []
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/dataset/learner/lazy/numeric.rb

@@ -0,0 +1,19 @@
+module EasyML
+  class Dataset
+    class Learner
+      class Lazy
+        class Numeric < Query
+          def train_query
+            super.concat([
+              Polars.col(column.name).mean.alias("#{column.name}__mean"),
+              Polars.col(column.name).median.alias("#{column.name}__median"),
+              Polars.col(column.name).min.alias("#{column.name}__min"),
+              Polars.col(column.name).max.alias("#{column.name}__max"),
+              Polars.col(column.name).std.alias("#{column.name}__std"),
+            ])
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/dataset/learner/lazy/query.rb

@@ -0,0 +1,69 @@
+module EasyML
+  class Dataset
+    class Learner
+      class Lazy
+        class Query < EasyML::Dataset::Learner::Query
+          def adapter
+            case dtype.to_sym
+            when :float, :integer
+              Lazy::Numeric
+            when :string, :text
+              Lazy::String
+            when :categorical
+              Lazy::Categorical
+            when :datetime, :date
+              Lazy::Datetime
+            when :boolean
+              Lazy::Boolean
+            when :null
+              Lazy::Null
+            else
+              raise "Don't know how to learn from dtype: #{dtype}"
+            end
+          end
+
+          def execute(split)
+            case split.to_sym
+            when :train
+              train_query
+            when :data
+              full_dataset_query
+            end
+          end
+
+          private
+
+          def full_dataset_query
+            [num_rows, null_count].compact
+          end
+
+          def train_query
+            [last_value, most_frequent_value].compact
+          end
+
+          def null_count
+            Polars.col(column.name).null_count.alias("#{column.name}__null_count")
+          end
+
+          def num_rows
+            Polars.col(column.name).len.alias("#{column.name}__num_rows")
+          end
+
+          def most_frequent_value
+            Polars.col(column.name).filter(Polars.col(column.name).is_not_null).mode.first.alias("#{column.name}__most_frequent_value")
+          end
+
+          def last_value
+            return unless dataset.date_column.present?
+
+            Polars.col(column.name)
+              .sort_by(dataset.date_column.name, reverse: true, nulls_last: true)
+              .filter(Polars.col(column.name).is_not_null)
+              .first
+              .alias("#{column.name}__last_value")
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/dataset/learner/lazy/string.rb

@@ -0,0 +1,19 @@
+module EasyML
+  class Dataset
+    class Learner
+      class Lazy
+        class String < Query
+          def full_dataset_query
+            super.concat([
+              unique_count,
+            ])
+          end
+
+          def unique_count
+            Polars.col(column.name).cast(:str).n_unique.alias("#{column.name}__unique_count")
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/dataset/learner/lazy.rb

@@ -0,0 +1,51 @@
+module EasyML
+  class Dataset
+    class Learner
+      class Lazy < Base
+        def learn
+          # types.map
+          types.reduce({}) do |h, type|
+            h.tap do
+              h[type] = learn_using_split(:train, type).deep_merge!(learn_using_split(:data, type))
+            end
+          end
+        end
+
+        private
+
+        def learn_using_split(split, type)
+          return {} if @dataset.send(type).empty?
+
+          get_column_statistics(run_queries(split, type))
+        end
+
+        def run_queries(split, type)
+          queries = build_queries(split, type)
+          @dataset.send(type).send(split, all_columns: true, lazy: true).select(queries).collect
+        end
+
+        def get_column_statistics(query_results)
+          query_results.columns.group_by { |k| k.split("__").first }.reduce({}) do |h, (k, v)|
+            h.tap do
+              h[k] ||= {}
+              v.each do |col|
+                statistic_name = col.split("__").last
+                h[k][statistic_name] = query_results[col][0]
+              end
+            end
+          end
+        end
+
+        def build_queries(split, type)
+          columns.flat_map do |column|
+            next if skip_processing?(column, type)
+
+            query = Lazy::Query.new(@dataset, column)
+            query_adapter = query.adapter.new(@dataset, column)
+            query_adapter.execute(split)
+          end.compact
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/dataset/learner/query.rb

@@ -0,0 +1,25 @@
+module EasyML
+  class Dataset
+    class Learner
+      class Query
+        attr_reader :dataset, :column, :dtype, :raw_dtype
+
+        def initialize(dataset, column)
+          @dataset = dataset
+          @column = column
+          begin
+            # TODO: LAZIFY THIS
+            @dtype = column.datatype || EasyML::Data::PolarsColumn.determine_type(column.raw.data[column.name])
+            @raw_dtype = column.raw_dtype
+          rescue => e
+            raise "Unable to find column #{column.name}. If this column is computed by a feature, you forgot to declare computes_columns"
+          end
+        end
+
+        def execute(split)
+          adapter.execute(split)
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/dataset/learner.rb

@@ -0,0 +1,100 @@
+module EasyML
+  class Dataset
+    class Learner
+      include EasyML::Timing
+      attr_accessor :dataset, :columns, :type, :computed, :raw_columns, :statistics
+
+      def initialize(dataset, type: :raw)
+        @dataset = dataset
+        @columns = dataset.columns.reload.needs_learn.sort_by(&:name)
+
+        if computed
+          @columns = @columns.computed
+        end
+
+        @columns = @columns.select(&:persisted?).reject(&:empty?)
+        @type = type
+      end
+
+      def learn
+        prepare
+        learn_statistics
+        save_statistics
+      end
+
+      private
+
+      def save_statistics
+        columns.each do |col|
+          col.merge_statistics(statistics.dig(col.name))
+          col.set_sample_values
+          col.assign_attributes(
+            learned_at: EasyML::Support::UTC.now,
+            last_datasource_sha: col.dataset.last_datasource_sha,
+            last_feature_sha: col.feature&.sha,
+            is_learning: type == :raw,
+          )
+        end
+
+        EasyML::Column.import(columns, on_duplicate_key_update: { columns: %i[
+                                         statistics
+                                         learned_at
+                                         sample_values
+                                         last_datasource_sha
+                                         is_learning
+                                       ] })
+        dataset.columns.set_feature_lineage(columns)
+      end
+
+      measure_method_timing :save_statistics
+
+      def learn_statistics
+        return @statistics if @statistics
+
+        @statistics = lazy_statistics.deep_merge!(eager_statistics).reduce({}) do |h, (type, stat_group)|
+          h.tap do
+            stat_group.each do |statistic, value|
+              h[statistic] ||= {}
+              h[statistic][type] = value
+            end
+          end
+        end.with_indifferent_access
+
+        if type != :raw
+          columns.select(&:one_hot?).each do |column|
+            @statistics[column.name][:processed] = @statistics[column.name][:raw]
+          end
+        end
+      end
+
+      measure_method_timing :learn_statistics
+
+      def prepare
+        @schema = EasyML::Data::PolarsSchema.simplify(@dataset.raw_schema).symbolize_keys
+        @raw_columns = @schema.keys.sort.map(&:to_s)
+        columns.each do |column|
+          attrs = {
+            in_raw_dataset: @raw_columns.include?(column.name),
+            datatype: column.read_attribute(:datatype).present? ? nil : @schema[column.name.to_sym],
+          }.compact
+          column.assign_attributes(attrs)
+        end
+        EasyML::Column.import(columns, on_duplicate_key_update: { columns: %i[in_raw_dataset datatype] })
+      end
+
+      measure_method_timing :prepare
+
+      def lazy_statistics
+        Lazy.new(dataset, columns, type: type).learn
+      end
+
+      measure_method_timing :lazy_statistics
+
+      def eager_statistics
+        Eager.new(dataset, columns, type: type).learn
+      end
+
+      measure_method_timing :eager_statistics
+    end
+  end
+end