easy_ml 0.2.0.pre.rc57 → 0.2.0.pre.rc60

data/app/models/easy_ml/dataset.rb

@@ -1,24 +1,24 @@
-# == Schetuma Information
+# == Schema Information
 #
 # Table name: easy_ml_datasets
 #
-#  id                      :bigint           not null, primary key
-#  name                    :string           not null
-#  description             :string
-#  dataset_type            :string
-#  status                  :string
-#  version                 :string
-#  datasource_id           :bigint
-#  root_dir                :string
-#  configuration           :json
-#  num_rows                :bigint
-#  workflow_status         :string
-#  statistics              :json
-#  preprocessor_statistics :json
-#  schema                  :json
-#  refreshed_at            :datetime
-#  created_at              :datetime         not null
-#  updated_at              :datetime         not null
+#  id                  :bigint           not null, primary key
+#  name                :string           not null
+#  description         :string
+#  dataset_type        :string
+#  status              :string
+#  version             :string
+#  datasource_id       :bigint
+#  root_dir            :string
+#  configuration       :json
+#  num_rows            :bigint
+#  workflow_status     :string
+#  statistics          :json
+#  schema              :json
+#  refreshed_at        :datetime
+#  created_at          :datetime         not null
+#  updated_at          :datetime         not null
+#  last_datasource_sha :string
 #
 module EasyML
   class Dataset < ActiveRecord::Base
@@ -45,7 +45,7 @@ module EasyML
     has_many :columns, class_name: "EasyML::Column", dependent: :destroy, extend: EasyML::ColumnList
     accepts_nested_attributes_for :columns, allow_destroy: true, update_only: true
 
-    has_many :features, dependent: :destroy, class_name: "EasyML::Feature"
+    has_many :features, dependent: :destroy, class_name: "EasyML::Feature", extend: EasyML::FeatureList
     accepts_nested_attributes_for :features, allow_destroy: true
 
     has_many :events, as: :eventable, class_name: "EasyML::Event", dependent: :destroy
@@ -80,7 +80,7 @@ module EasyML
         column_types: EasyML::Data::PolarsColumn::TYPE_MAP.keys.map do |type|
           { value: type.to_s, label: type.to_s.titleize }
         end,
-        preprocessing_strategies: EasyML::Data::Preprocessor.constants[:preprocessing_strategies],
+        preprocessing_strategies: EasyML::Column::Imputers.constants[:preprocessing_strategies],
         feature_options: EasyML::Features::Registry.list_flat,
         splitter_constants: EasyML::Splitter.constants,
       }
@@ -119,13 +119,6 @@ module EasyML
       processed.data(limit: 1)&.schema || raw.data(limit: 1)&.schema
     end
 
-    def refresh_datatypes
-      return unless columns_need_refresh?
-
-      cleanup
-      datasource.reread(columns)
-    end
-
     def num_rows
       if datasource&.num_rows.nil?
         datasource.after_sync
@@ -140,6 +133,12 @@ module EasyML
       EasyML::RefreshDatasetJob.perform_later(id)
     end
 
+    def best_segment
+      [processed, raw].detect do |segment|
+        segment.send(:data, all_columns: true, limit: 1)&.columns
+      end
+    end
+
     def raw
       return @raw if @raw && @raw.dataset
 
@@ -162,22 +161,36 @@ module EasyML
       save
     end
 
+    def refreshed_datasource?
+      last_datasource_sha_changed?
+    end
+
+    def prepare_features
+      features.update_all(workflow_status: "ready")
+    end
+
     def prepare!
+      prepare_features
       cleanup
       refresh_datasource!
       split_data
+      process_data
     end
 
     def prepare
+      prepare_features
       refresh_datasource
       split_data
+      process_data
     end
 
     def actually_refresh
       refreshing do
+        learn(delete: false) # After syncing datasource, learn new statistics + sync columns
         process_data
         fully_reload
         learn
+        learn_statistics(type: :processed) # After processing data, we learn any new statistics
         now = UTC.now
         update(workflow_status: "ready", refreshed_at: now, updated_at: now)
         fully_reload
@@ -245,19 +258,57 @@ module EasyML
       features_need_fit.any?
     end
 
-    def refresh_reasons
+    # Some of these are expensive to calculate, so we only want to include
+    # them in the refresh reasons if they are actually needed.
+    #
+    # During dataset_serializer for instance, we don't want to check s3,
+    # we only do that during background jobs.
+    #
+    # So yes this is an annoying way to structure a method, but it's helpful for performance
+    #
+    def refresh_reasons(exclude: [])
       {
-        "Not split" => not_split?,
-        "Refreshed at is nil" => refreshed_at.nil?,
-        "Columns need refresh" => columns_need_refresh?,
-        "Features need refresh" => features_need_fit?,
-        "Datasource needs refresh" => datasource_needs_refresh?,
-        "Datasource was refreshed" => datasource_was_refreshed?,
-      }.select { |k, v| v }.map { |k, v| k }
+        not_split: {
+          name: "Not split",
+          check: -> { not_split? },
+        },
+        refreshed_at_is_nil: {
+          name: "Refreshed at is nil",
+          check: -> { refreshed_at.nil? },
+        },
+        columns_need_refresh: {
+          name: "Columns need refresh",
+          check: -> { columns_need_refresh? },
+        },
+        features_need_fit: {
+          name: "Features need fit",
+          check: -> { features_need_fit? },
+        },
+        datasource_needs_refresh: {
+          name: "Datasource needs refresh",
+          check: -> { datasource_needs_refresh? },
+        },
+        refreshed_datasource: {
+          name: "Refreshed datasource",
+          check: -> { refreshed_datasource? },
+        },
+        datasource_was_refreshed: {
+          name: "Datasource was refreshed",
+          check: -> { datasource_was_refreshed? },
+        },
+      }.except(*exclude).select do |k, config|
+        config[:check].call
+      end.map do |k, config|
+        config[:name]
+      end
     end
 
-    def needs_refresh?
-      refresh_reasons.any?
+    def needs_refresh?(exclude: [])
+      refresh_reasons(exclude: exclude).any?
+    end
+
+    def processed?
+      !needs_refresh?
     end
 
     def not_split?
@@ -274,7 +325,6 @@ module EasyML
 
     def learn(delete: true)
       learn_schema
-      learn_statistics
       columns.sync(delete: delete)
     end
 
@@ -326,6 +376,8 @@ module EasyML
 
     def learn_schema
       data = processed.data(limit: 1).to_a.any? ? processed.data : raw.data
+      return nil if data.nil?
+
       schema = data.schema.reduce({}) do |h, (k, v)|
         h.tap do
           h[k] = EasyML::Data::PolarsColumn.polars_to_sym(v)
@@ -334,23 +386,23 @@ module EasyML
       write_attribute(:schema, schema)
     end
 
-    def learn_statistics
-      stats = {
-        raw: EasyML::Data::StatisticsLearner.learn(raw, self),
-      }
-      stats.merge!(processed: EasyML::Data::StatisticsLearner.learn(processed, self)) if processed.data.present?
+    def learn_statistics(type: :raw, computed: false)
+      columns.learn(type: type, computed: computed)
+      update(
+        statistics: columns.reload.statistics,
+      )
+    end
 
-      update(statistics: stats)
+    def statistics
+      (read_attribute(:statistics) || {}).with_indifferent_access
     end
 
     def process_data
-      split_data
       fit
       normalize_all
-      # alert_nulls
     end
 
-    def needs_learn?(df)
+    def needs_learn?
       return true if columns_need_refresh?
 
       never_learned = columns.none?
@@ -359,6 +411,7 @@ module EasyML
       new_features = features.any? { |f| f.updated_at > columns.maximum(:updated_at) }
       return true if new_features
 
+      df = raw.query(limit: 1)
       new_cols = df.present? ? (df.columns - columns.map(&:name)) : []
       new_cols = columns.syncable
 
@@ -390,22 +443,23 @@ module EasyML
       { differing_columns: differing_columns, differences: differences }
     end
 
-    def normalize(df = nil, split_ys: false, inference: false, all_columns: false, features: self.features, idx: nil)
-      df = apply_features(df, features)
-      df = drop_nulls(df)
-      df = apply_missing_features(df, inference: inference)
-      df = preprocessor.postprocess(df, inference: inference)
+    def validate_input(df)
+      fields = missing_required_fields(df)
+      return fields.empty?, fields
+    end
 
-      # Learn will update columns, so if any features have been added
-      # since the last time columns were learned, we should re-learn the schema
-      learn(delete: false) if idx == 1 && needs_learn?(df)
+    def normalize(df = nil, split_ys: false, inference: false, all_columns: false, features: self.features)
+      df = apply_missing_features(df, inference: inference)
+      df = drop_nulls(df)
+      df = columns.transform(df, inference: inference)
+      df = apply_features(df, features)
+      df = columns.transform(df, inference: inference, computed: true)
       df = apply_column_mask(df, inference: inference) unless all_columns
-      raise_on_nulls(df) if inference
       df, = processed.split_features_targets(df, true, target) if split_ys
       df
     end
 
-    def raise_on_nulls(df)
+    def missing_required_fields(df)
       desc_df = df.describe
 
       # Get the 'null_count' row
@@ -416,8 +470,10 @@ module EasyML
         null_count_row[col][0].to_i > 0
       end
 
-      if columns_with_nulls.any?
-        raise "Null values found in columns: #{columns_with_nulls.join(", ")}"
+      # This is a history class, because this only occurs on prediction
+      required_columns = columns.current.required.map(&:name)
+      required_columns.select do |col|
+        columns_with_nulls.include?(col) || df.columns.map(&:to_s).exclude?(col.to_s)
       end
     end
 
@@ -478,16 +534,15 @@ module EasyML
       result.empty? ? nil : result
     end
 
-    def processed?
-      !should_split?
-    end
-
     def decode_labels(ys, col: nil)
-      preprocessor.decode_labels(ys, col: col.nil? ? target : col)
+      if col.nil?
+        col = target
+      end
+      preloaded_columns.find_by(name: col).decode_labels(ys)
     end
 
     def preprocessing_steps
-      return if columns.nil? || (columns.respond_to?(:empty?) && columns.empty?)
+      return {} if preloaded_columns.nil? || (preloaded_columns.respond_to?(:empty?) && preloaded_columns.empty?)
       return @preprocessing_steps if @preprocessing_steps.present?
 
       training = standardize_preprocessing_steps(:training)
@@ -499,23 +554,16 @@ module EasyML
       }.compact.deep_symbolize_keys
     end
 
-    def preprocessor
-      @preprocessor ||= initialize_preprocessor
-      return @preprocessor if @preprocessor.preprocessing_steps == preprocessing_steps
-
-      @preprocessor = initialize_preprocessor
-    end
-
     def target
       @target ||= preloaded_columns.find(&:is_target)&.name
     end
 
     def date_column
-      @date_column ||= columns.date_column.first
+      @date_column ||= preloaded_columns.find(&:is_date_column?)
     end
 
     def drop_cols
-      @drop_cols ||= preloaded_columns.select(&:hidden).flat_map(&:columns)
+      @drop_cols ||= preloaded_columns.select(&:hidden).flat_map(&:aliases)
     end
 
     def drop_if_null
@@ -552,10 +600,14 @@ module EasyML
       df[column_mask(df, inference: inference)]
     end
 
-    def apply_missing_features(df, inference: false)
+    def apply_missing_features(df, inference: false, include_one_hots: false)
       return df unless inference
 
       missing_features = (col_order(inference: inference) - df.columns).compact
+      unless include_one_hots
+        missing_features -= columns.one_hots.flat_map(&:virtual_columns) unless include_one_hots
+        missing_features += columns.one_hots.map(&:name) - df.columns
+      end
       df.with_columns(missing_features.map { |f| Polars.lit(nil).alias(f) })
     end
 
@@ -576,7 +628,7 @@ module EasyML
     end
 
     def upload_remote_files
-      return unless processed?
+      return if !needs_refresh?
 
       processed.upload.tap do
         features.each(&:upload_remote_files)
@@ -648,27 +700,42 @@ module EasyML
 
     def refresh_datasource
       datasource.reload.refresh
-      refresh_datatypes
-      initialize_splits
+      after_refresh_datasource
     end
 
     def refresh_datasource!
       datasource.reload.refresh!
-      refresh_datatypes
+      after_refresh_datasource
+    end
+
+    def after_refresh_datasource
+      update(last_datasource_sha: datasource.sha)
       initialize_splits
     end
 
     def normalize_all
       processed.cleanup
 
-      SPLIT_ORDER.each_with_index do |segment, idx|
+      SPLIT_ORDER.each do |segment|
         df = raw.read(segment)
-        processed_df = normalize(df, all_columns: true, idx: idx)
+        learn_computed_columns(df) if segment == :train
+        processed_df = normalize(df, all_columns: true)
         processed.save(segment, processed_df)
       end
       @normalized = true
     end
 
+    def learn_computed_columns(df)
+      return unless features.ready_to_apply.any?
+
+      df = df.clone
+      df = apply_features(df)
+      processed.save(:train, df)
+      learn(delete: false)
+      learn_statistics(type: :processed, computed: true)
+      processed.cleanup
+    end
+
     def drop_nulls(df)
       return df if drop_if_null.nil? || drop_if_null.empty?
 
@@ -678,8 +745,12 @@ module EasyML
       df.drop_nulls(subset: drop)
     end
 
+    # Pass refresh: false for frontend views so we don't query S3 during web requests
     def load_data(segment, **kwargs, &block)
-      if processed?
+      needs_refresh = kwargs.key?(:refresh) ? kwargs[:refresh] : needs_refresh?
+      kwargs.delete(:refresh)
+
+      if !needs_refresh
         processed.load_data(segment, **kwargs, &block)
       else
         raw.load_data(segment, **kwargs, &block)
@@ -687,8 +758,7 @@ module EasyML
     end
 
     def fit
-      preprocessor.fit(raw.train(all_columns: true))
-      self.preprocessor_statistics = preprocessor.statistics
+      learn_statistics(type: :raw)
     end
 
     # log_method :fit, "Learning statistics", verbose: true
@@ -698,10 +768,9 @@ module EasyML
     end
 
     def split_data(force: false)
-      return unless force || should_split?
+      return unless force || needs_refresh?
 
       cleanup
-      features = self.features.ordered.load
       splitter.split(datasource) do |train_df, valid_df, test_df|
         [:train, :valid, :test].zip([train_df, valid_df, test_df]).each do |segment, df|
           raw.save(segment, df)
@@ -709,10 +778,6 @@ module EasyML
       end
     end
 
-    def should_split?
-      needs_refresh?
-    end
-
     def filter_duplicate_features
       return unless attributes["features_attributes"].present?
 
@@ -733,6 +798,7 @@ module EasyML
     end
 
     def apply_features(df, features = self.features)
+      features = features.ready_to_apply
       if features.nil? || features.empty?
         df
       else
@@ -754,10 +820,6 @@ module EasyML
 
           result = feature.transform_batch(acc_df)
 
-          unless result.is_a?(Polars::DataFrame)
-            raise "Feature '#{feature.name}' must return a Polars::DataFrame, got #{result.class}"
-          end
-
           result
         end
       end
@@ -769,16 +831,6 @@ module EasyML
       end).to_h.compact.reject { |_k, v| v["method"] == "none" }
     end
 
-    def initialize_preprocessor
-      EasyML::Data::Preprocessor.new(
-        directory: Pathname.new(root_dir).append("preprocessor"),
-        preprocessing_steps: preprocessing_steps,
-        dataset: self,
-      ).tap do |preprocessor|
-        preprocessor.statistics = preprocessor_statistics
-      end
-    end
-
     def fully_reload
       return unless persisted?
 

data/app/models/easy_ml/dataset_history.rb

@@ -2,28 +2,28 @@
 #
 # Table name: easy_ml_dataset_histories
 #
-#  id                      :bigint           not null, primary key
-#  dataset_id              :integer          not null
-#  name                    :string           not null
-#  description             :string
-#  dataset_type            :string
-#  status                  :string
-#  version                 :string
-#  datasource_id           :integer
-#  root_dir                :string
-#  configuration           :json
-#  num_rows                :integer
-#  workflow_status         :string
-#  statistics              :json
-#  preprocessor_statistics :json
-#  schema                  :json
-#  refreshed_at            :datetime
-#  created_at              :datetime         not null
-#  updated_at              :datetime         not null
-#  history_started_at      :datetime         not null
-#  history_ended_at        :datetime
-#  history_user_id         :integer
-#  snapshot_id             :string
+#  id                  :bigint           not null, primary key
+#  dataset_id          :integer          not null
+#  name                :string           not null
+#  description         :string
+#  dataset_type        :string
+#  status              :string
+#  version             :string
+#  datasource_id       :integer
+#  root_dir            :string
+#  configuration       :json
+#  num_rows            :integer
+#  workflow_status     :string
+#  statistics          :json
+#  schema              :json
+#  refreshed_at        :datetime
+#  created_at          :datetime         not null
+#  updated_at          :datetime         not null
+#  history_started_at  :datetime         not null
+#  history_ended_at    :datetime
+#  history_user_id     :integer
+#  snapshot_id         :string
+#  last_datasource_sha :string
 #
 module EasyML
   class DatasetHistory < ActiveRecord::Base
@@ -44,7 +44,7 @@ module EasyML
       true
     end
 
-    def should_split?
+    def needs_refresh?
       false
     end
   end

data/app/models/easy_ml/datasource.rb

@@ -10,6 +10,7 @@
 #  refreshed_at    :datetime
 #  created_at      :datetime         not null
 #  updated_at      :datetime         not null
+#  sha             :string
 #
 module EasyML
   class Datasource < ActiveRecord::Base
@@ -55,6 +56,7 @@ module EasyML
 
     has_many :events, as: :eventable, class_name: "EasyML::Event", dependent: :destroy
     attr_accessor :schema, :columns, :num_rows, :is_syncing
+    belongs_to :dataset, class_name: "EasyML::Dataset", optional: true, dependent: :destroy
 
     add_configuration_attributes :schema, :columns, :num_rows, :polars_args, :verbose, :is_syncing
     DATASOURCE_CONSTANTS.flat_map(&:configuration_attributes).each do |attribute|
@@ -118,11 +120,13 @@ module EasyML
       self.num_rows = data.shape[0]
       self.is_syncing = false
       self.refreshed_at = Time.now
+      self.sha = adapter.sha
       save
     end
 
     def refresh
       unless adapter.needs_refresh?
+        update(sha: adapter.sha) if sha.nil?
         update!(is_syncing: false)
         return
       end

data/app/models/easy_ml/datasource_history.rb

@@ -15,6 +15,7 @@
 #  history_ended_at   :datetime
 #  history_user_id    :integer
 #  snapshot_id        :string
+#  sha                :string
 #
 module EasyML
   class DatasourceHistory < ActiveRecord::Base

data/app/models/easy_ml/datasources/file_datasource.rb

@@ -1,7 +1,7 @@
 module EasyML
   module Datasources
     class FileDatasource < BaseDatasource
-      delegate :query, :convert_to_parquet, to: :reader
+      delegate :query, :convert_to_parquet, :sha, to: :reader
 
       def after_sync
         reader.normalize

data/app/models/easy_ml/datasources/polars_datasource.rb

@@ -6,18 +6,8 @@ module EasyML
       validates :df, presence: true
       add_configuration_attributes :df
 
-      def query(drop_cols: [], filter: nil, limit: nil, select: nil, unique: nil, sort: nil, descending: false)
-        return if df.nil?
-
-        df = self.df.clone
-        df = df.filter(filter) if filter
-        df = df.select(select) if select.present?
-        df = df.unique if unique
-        drop_cols &= df.columns
-        df = df.drop(drop_cols) unless drop_cols.empty?
-        df = df.sort(sort, reverse: descending) if sort
-        df = df.limit(limit) if limit
-        df
+      def query(**kwargs)
+        EasyML::Data::PolarsInMemory.query(df, **kwargs)
       end
 
       def in_batches(of: 10_000)
@@ -40,6 +30,10 @@ module EasyML
         datasource.updated_at
       end
 
+      def sha
+        nil
+      end
+
       def data
         df
       end

data/app/models/easy_ml/datasources/s3_datasource.rb

@@ -17,7 +17,7 @@ module EasyML
       add_configuration_attributes :s3_bucket, :s3_prefix, :s3_region, :cache_for
 
       delegate :query, :data, :s3_access_key_id, :s3_secret_access_key, :before_sync, :after_sync, :clean,
-               to: :synced_directory
+               :sha, to: :synced_directory
 
       def in_batches(&block)
         synced_directory.in_batches(&block)