easy_ml 0.2.0.pre.rc58 → 0.2.0.pre.rc61

data/app/models/easy_ml/column/learners/null.rb

@@ -0,0 +1,22 @@
+module EasyML
+  class Column
+    module Learners
+      class Null < Base
+        def full_dataset_statistics(df)
+          return {} if df.nil?
+
+          {
+            num_rows: df.size,
+            null_count: df[column.name].null_count || 0,
+          }
+        end
+
+        def train_statistics(df)
+          {
+            last_value: last_value(df),
+          }
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/learners/numeric.rb

@@ -0,0 +1,33 @@
+module EasyML
+  class Column
+    module Learners
+      class Numeric < Base
+        def train_statistics(df)
+          return {} if df.nil?
+
+          super(df).merge!({
+            mean: df[column.name].mean,
+            median: df[column.name].median,
+            min: df[column.name].min,
+            max: df[column.name].max,
+            std: df[column.name].std,
+            last_value: last_value(df),
+          }.compact)
+        end
+
+        def last_value(df)
+          if dataset.date_column.present?
+            sorted_df = df.sort(dataset.date_column.name, reverse: true)
+            last_value = sorted_df
+              .filter(Polars.col(column.name).is_not_null)
+              .select(column.name)
+              .head(1)
+              .item
+
+            last_value
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/learners/string.rb

@@ -0,0 +1,15 @@
+module EasyML
+  class Column
+    module Learners
+      class String < Base
+        def full_dataset_statistics(df)
+          return {} if df.nil?
+
+          super(df).merge!({
+            unique_count: df[column.name].cast(:str).n_unique,
+          })
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/lineage/base.rb

@@ -0,0 +1,22 @@
+module EasyML
+  class Column
+    class Lineage
+      class Base
+        attr_accessor :dataset, :column
+
+        def initialize(column)
+          @column = column
+          @dataset = column.dataset
+        end
+
+        def as_json
+          {
+            key: key,
+            description: description,
+            timestamp: timestamp,
+          }.with_indifferent_access
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/lineage/computed_by_feature.rb

@@ -0,0 +1,23 @@
+module EasyML
+  class Column
+    class Lineage
+      class ComputedByFeature < Base
+        def key
+          :computed_by_feature
+        end
+
+        def description
+          "Computed by #{column.computed_by}"
+        end
+
+        def timestamp
+          column.feature.fit_at
+        end
+
+        def check
+          column.computed_by.present?
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/lineage/preprocessed.rb

@@ -0,0 +1,23 @@
+module EasyML
+  class Column
+    class Lineage
+      class Preprocessed < Base
+        def key
+          :preprocessed
+        end
+
+        def description
+          "Preprocessed using #{column.imputers.preprocessing_descriptions.join(", ")}"
+        end
+
+        def timestamp
+          column.dataset.refreshed_at
+        end
+
+        def check
+          column.preprocessing_steps.present?
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/lineage/raw_dataset.rb

@@ -0,0 +1,23 @@
+module EasyML
+  class Column
+    class Lineage
+      class RawDataset < Base
+        def key
+          :raw_dataset
+        end
+
+        def description
+          "Present in raw dataset"
+        end
+
+        def timestamp
+          column.dataset.datasource.refreshed_at
+        end
+
+        def check
+          column.in_raw_dataset?
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/lineage.rb

@@ -0,0 +1,28 @@
+module EasyML
+  class Column
+    class Lineage
+      attr_accessor :dataset, :column
+
+      def initialize(column)
+        @column = column
+        @dataset = column.dataset
+      end
+
+      def sort_order
+        [
+          RawDataset,
+          ComputedByFeature,
+          Preprocessed,
+        ]
+      end
+
+      def lineage
+        sort_order.map do |cl|
+          cl.new(column)
+        end.select(&:check)
+          .map(&:as_json)
+          .compact
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/selector.rb

@@ -0,0 +1,96 @@
+module EasyML
+  class Column
+    class Selector
+      attr_accessor :selected, :dataset, :column, :transform
+
+      def initialize(column, selected = nil, &block)
+        @column = column
+        @dataset = column.dataset
+        @selected = selected
+        @transform = block
+      end
+
+      def name
+        column.name
+      end
+
+      def raw
+        if column.is_computed? && !column.in_raw_dataset?
+          Selector.new(column, :processed)
+        else
+          Selector.new(column, :raw)
+        end
+      end
+
+      def clipped
+        Selector.new(column, :raw) do |df|
+          column.imputers.training.clip(df)
+        end
+      end
+
+      def processed
+        Selector.new(column, :processed)
+      end
+
+      def train(**kwargs)
+        select(:train, **kwargs)
+      end
+
+      def test(**kwargs)
+        select(:test, **kwargs)
+      end
+
+      def valid(**kwargs)
+        select(:valid, **kwargs)
+      end
+
+      def data(**kwargs)
+        if column.is_computed?
+          Selector.new(column, :processed).send(:select, :data, **kwargs)
+        else
+          select(:data, **kwargs)
+        end
+      end
+
+      private
+
+      def select(segment, **orig_kwargs)
+        kwargs = orig_kwargs.clone
+        return nil if dataset.nil?
+
+        kwargs[:all_columns] = true
+
+        if kwargs.key?(:select)
+          kwargs[:select] = [kwargs[:select]].flatten
+        else
+          kwargs[:select] = []
+        end
+
+        if (selected == :processed || (selected.nil? && !dataset.needs_refresh?)) && column.one_hot?
+          kwargs[:select] << column.virtual_columns
+        else
+          kwargs[:select] << column.name
+        end
+        kwargs[:select] = kwargs[:select].flatten.uniq
+
+        if @selected.present?
+          available_columns = dataset.send(@selected).send(segment, limit: 1, all_columns: true)&.columns || []
+          kwargs[:select] = available_columns & kwargs[:select]
+          return Polars::DataFrame.new if kwargs[:select].empty?
+          result = dataset.send(@selected).send(segment, **kwargs)
+        else
+          available_columns = dataset.send(segment, limit: 1, all_columns: true)&.columns || []
+          kwargs[:select] = available_columns & kwargs[:select]
+          return Polars::DataFrame.new if kwargs[:select].empty?
+          result = dataset.send(segment, **kwargs)
+        end
+
+        if transform
+          result = transform.call(result)
+        end
+
+        result
+      end
+    end
+  end
+end