easy_ml 0.2.0.pre.rc58 → 0.2.0.pre.rc61

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: easy_ml
 version: !ruby/object:Gem::Version
-  version: 0.2.0.pre.rc58
+  version: 0.2.0.pre.rc61
 platform: ruby
 authors:
 - Brett Shollenberger
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-02-02 00:00:00.000000000 Z
+date: 2025-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -559,10 +559,36 @@ files:
 - app/jobs/easy_ml/sync_datasource_job.rb
 - app/jobs/easy_ml/training_job.rb
 - app/models/concerns/easy_ml/dataframe_serialization.rb
-- app/models/easy_ml/adapters/base_adapter.rb
-- app/models/easy_ml/adapters/polars_adapter.rb
 - app/models/easy_ml/cleaner.rb
 - app/models/easy_ml/column.rb
+- app/models/easy_ml/column/imputers.rb
+- app/models/easy_ml/column/imputers/base.rb
+- app/models/easy_ml/column/imputers/categorical.rb
+- app/models/easy_ml/column/imputers/clip.rb
+- app/models/easy_ml/column/imputers/constant.rb
+- app/models/easy_ml/column/imputers/ffill.rb
+- app/models/easy_ml/column/imputers/imputer.rb
+- app/models/easy_ml/column/imputers/mean.rb
+- app/models/easy_ml/column/imputers/median.rb
+- app/models/easy_ml/column/imputers/most_frequent.rb
+- app/models/easy_ml/column/imputers/null_imputer.rb
+- app/models/easy_ml/column/imputers/one_hot_encoder.rb
+- app/models/easy_ml/column/imputers/ordinal_encoder.rb
+- app/models/easy_ml/column/imputers/today.rb
+- app/models/easy_ml/column/learner.rb
+- app/models/easy_ml/column/learners/base.rb
+- app/models/easy_ml/column/learners/boolean.rb
+- app/models/easy_ml/column/learners/categorical.rb
+- app/models/easy_ml/column/learners/datetime.rb
+- app/models/easy_ml/column/learners/null.rb
+- app/models/easy_ml/column/learners/numeric.rb
+- app/models/easy_ml/column/learners/string.rb
+- app/models/easy_ml/column/lineage.rb
+- app/models/easy_ml/column/lineage/base.rb
+- app/models/easy_ml/column/lineage/computed_by_feature.rb
+- app/models/easy_ml/column/lineage/preprocessed.rb
+- app/models/easy_ml/column/lineage/raw_dataset.rb
+- app/models/easy_ml/column/selector.rb
 - app/models/easy_ml/column_history.rb
 - app/models/easy_ml/column_list.rb
 - app/models/easy_ml/concerns/configurable.rb
@@ -580,6 +606,7 @@ files:
 - app/models/easy_ml/event_context.rb
 - app/models/easy_ml/feature.rb
 - app/models/easy_ml/feature_history.rb
+- app/models/easy_ml/feature_list.rb
 - app/models/easy_ml/model.rb
 - app/models/easy_ml/model_file.rb
 - app/models/easy_ml/model_file_history.rb
@@ -628,6 +655,7 @@ files:
 - bin/setup
 - bin/vite
 - config/initializers/dataframe.rb
+- config/initializers/enumerable.rb
 - config/initializers/evaluators.rb
 - config/initializers/inflections.rb
 - config/initializers/resque.rb
@@ -654,15 +682,13 @@ files:
 - lib/easy_ml/data/date_converter.rb
 - lib/easy_ml/data/filter_extensions.rb
 - lib/easy_ml/data/polars_column.rb
+- lib/easy_ml/data/polars_in_memory.rb
 - lib/easy_ml/data/polars_reader.rb
-- lib/easy_ml/data/preprocessor.rb
 - lib/easy_ml/data/preprocessor/utils.rb
-- lib/easy_ml/data/simple_imputer.rb
 - lib/easy_ml/data/splits.rb
 - lib/easy_ml/data/splits/file_split.rb
 - lib/easy_ml/data/splits/in_memory_split.rb
 - lib/easy_ml/data/splits/split.rb
-- lib/easy_ml/data/statistics_learner.rb
 - lib/easy_ml/data/synced_directory.rb
 - lib/easy_ml/data/utils.rb
 - lib/easy_ml/engine.rb
@@ -677,6 +703,9 @@ files:
 - lib/easy_ml/railtie/templates/migration/add_computed_columns_to_easy_ml_columns.rb.tt
 - lib/easy_ml/railtie/templates/migration/add_default_to_is_target.rb.tt
 - lib/easy_ml/railtie/templates/migration/add_is_date_column_to_easy_ml_columns.rb.tt
+- lib/easy_ml/railtie/templates/migration/add_last_feature_sha_to_columns.rb.tt
+- lib/easy_ml/railtie/templates/migration/add_learned_at_to_easy_ml_columns.rb.tt
+- lib/easy_ml/railtie/templates/migration/add_sha_to_datasources_datasets_and_columns.rb.tt
 - lib/easy_ml/railtie/templates/migration/add_slug_to_easy_ml_models.rb.tt
 - lib/easy_ml/railtie/templates/migration/add_workflow_status_to_easy_ml_features.rb.tt
 - lib/easy_ml/railtie/templates/migration/create_easy_ml_column_histories.rb.tt
@@ -701,6 +730,7 @@ files:
 - lib/easy_ml/railtie/templates/migration/create_easy_ml_splitters.rb.tt
 - lib/easy_ml/railtie/templates/migration/create_easy_ml_tuner_jobs.rb.tt
 - lib/easy_ml/railtie/templates/migration/drop_path_from_easy_ml_model_files.rb.tt
+- lib/easy_ml/railtie/templates/migration/remove_preprocessor_statistics_from_easy_ml_datasets.rb.tt
 - lib/easy_ml/support.rb
 - lib/easy_ml/support/age.rb
 - lib/easy_ml/support/est.rb
@@ -712,14 +742,15 @@ files:
 - lib/easy_ml/support/synced_file.rb
 - lib/easy_ml/support/utc.rb
 - lib/easy_ml/version.rb
+- lib/tasks/profile.rake
 - lib/tasks/resque.rake
 - lib/tasks/vite.rake
 - lib/tasks/zhong.rake
 - public/easy_ml/assets/.vite/manifest-assets.json
 - public/easy_ml/assets/.vite/manifest.json
-- public/easy_ml/assets/assets/Application-BUsRR6b6.css
-- public/easy_ml/assets/assets/entrypoints/Application.tsx-DmkdJsDd.js
-- public/easy_ml/assets/assets/entrypoints/Application.tsx-DmkdJsDd.js.map
+- public/easy_ml/assets/assets/Application-BbFobaXt.css
+- public/easy_ml/assets/assets/entrypoints/Application.tsx-Dni_GM8r.js
+- public/easy_ml/assets/assets/entrypoints/Application.tsx-Dni_GM8r.js.map
 homepage: https://github.com/brettshollenberger/easy_ml
 licenses:
 - MIT

data/app/models/easy_ml/adapters/base_adapter.rb

@@ -1,45 +0,0 @@
-module EasyML
-  module Adapters
-    class BaseAdapter
-      attr_reader :datasource
-
-      def initialize(datasource)
-        @datasource = datasource
-      end
-
-      def query(*)
-        raise NotImplementedError
-      end
-
-      def in_batches(*)
-        raise NotImplementedError
-      end
-
-      def files
-        raise NotImplementedError
-      end
-
-      def last_updated_at
-        raise NotImplementedError
-      end
-
-      def data
-        raise NotImplementedError
-      end
-
-      def needs_refresh?
-        false
-      end
-
-      def refresh
-        datasource.syncing do
-          # Default implementation does nothing
-        end
-      end
-
-      def refresh!
-        refresh
-      end
-    end
-  end
-end

data/app/models/easy_ml/adapters/polars_adapter.rb

@@ -1,77 +0,0 @@
-module EasyML
-  module Adapters
-    class PolarsAdapter < BaseAdapter
-      def initialize(datasource)
-        super
-        read_df_from_configuration
-      end
-
-      def query(drop_cols: [], filter: nil, limit: nil, select: nil, unique: nil, sort: nil, descending: false)
-        return if df.nil?
-
-        df = self.df.clone
-        df = df.filter(filter) if filter
-        df = df.select(select) if select.present?
-        df = df.unique if unique
-        drop_cols &= df.columns
-        df = df.drop(drop_cols) unless drop_cols.empty?
-        df = df.sort(sort, reverse: descending) if sort
-        df = df.limit(limit) if limit
-        df
-      end
-
-      def in_batches(of: 10_000)
-        total_rows = df.shape[0]
-        (0...total_rows).step(of) do |start|
-          end_index = [start + of, total_rows].min
-          yield df.slice(start, end_index - start)
-        end
-      end
-
-      def files
-        []
-      end
-
-      def last_updated_at
-        datasource.updated_at
-      end
-
-      def data
-        df
-      end
-
-      private
-
-      attr_accessor :df
-
-      def store_df_in_configuration
-        return unless df
-
-        datasource.configuration = (datasource.configuration || {}).merge(
-          "df" => JSON.parse(df.write_json)
-        )
-      end
-
-      def read_df_from_configuration
-        return unless datasource.configuration&.key?("df")
-
-        df_data = datasource.configuration["df"]
-        columns = df_data["columns"].map do |col|
-          dtype = case col["datatype"]
-                  when Hash
-                    if col["datatype"]["Datetime"]
-                      Polars::Datetime.new(col["datatype"]["Datetime"][0].downcase.to_sym).class
-                    else
-                      Polars::Utf8
-                    end
-                  else
-                    Polars.const_get(col["datatype"])
-                  end
-          Polars::Series.new(col["name"], col["values"], dtype: dtype)
-        end
-
-        @df = Polars::DataFrame.new(columns)
-      end
-    end
-  end
-end

data/lib/easy_ml/data/preprocessor.rb

@@ -1,340 +0,0 @@
-require "fileutils"
-require "polars"
-require "date"
-require "json"
-require_relative "simple_imputer"
-
-module EasyML::Data
-  class Preprocessor
-    CATEGORICAL_COMMON_MIN = 50
-
-    ALLOWED_PARAMS = {
-      constant: [:constant],
-      categorical: %i[categorical_min one_hot ordinal_encoding],
-      most_frequent: %i[one_hot ordinal_encoding],
-      mean: [:clip],
-      median: [:clip],
-    }
-
-    PREPROCESSING_STRATEGIES = {
-      float: [
-        { value: "ffill", label: "Forward Fill" },
-        { value: "mean", label: "Mean" },
-        { value: "median", label: "Median" },
-        { value: "constant", label: "Constant Value" },
-      ],
-      integer: [
-        { value: "ffill", label: "Forward Fill" },
-        { value: "mean", label: "Mean" },
-        { value: "median", label: "Median" },
-        { value: "constant", label: "Constant Value" },
-      ],
-      boolean: [
-        { value: "ffill", label: "Forward Fill" },
-        { value: "most_frequent", label: "Most Frequent" },
-        { value: "constant", label: "Constant Value" },
-      ],
-      datetime: [
-        { value: "ffill", label: "Forward Fill" },
-        { value: "constant", label: "Constant Value" },
-        { value: "today", label: "Current Date" },
-      ],
-      string: [
-        { value: "ffill", label: "Forward Fill" },
-        { value: "most_frequent", label: "Most Frequent" },
-        { value: "constant", label: "Constant Value" },
-      ],
-      text: [
-        { value: "ffill", label: "Forward Fill" },
-        { value: "most_frequent", label: "Most Frequent" },
-        { value: "constant", label: "Constant Value" },
-      ],
-      categorical: [
-        { value: "ffill", label: "Forward Fill" },
-        { value: "categorical", label: "Categorical" },
-        { value: "most_frequent", label: "Most Frequent" },
-        { value: "constant", label: "Constant Value" },
-      ],
-    }.freeze
-
-    attr_accessor :directory, :verbose, :imputers, :preprocessing_steps, :dataset
-    attr_reader :statistics
-
-    def initialize(options = {})
-      @directory = options[:directory]
-      @verbose = options[:verbose]
-      @imputers = options[:imputers]
-      @preprocessing_steps = options[:preprocessing_steps]
-      @dataset = options[:dataset]
-      @statistics = {}
-    end
-
-    def statistics=(stats)
-      @statistics = (stats || {}).deep_symbolize_keys
-    end
-
-    def apply_clip(df, preprocessing_steps)
-      df = df.clone
-      preprocessing_steps ||= {}
-      preprocessing_steps.deep_symbolize_keys!
-
-      (preprocessing_steps[:training] || {}).each_key do |col|
-        clip_params = preprocessing_steps.dig(:training, col, :params, :clip)
-        next unless clip_params
-
-        min = clip_params[:min]
-        max = clip_params[:max]
-        df[col.to_s] = df[col.to_s].clip(min, max)
-      end
-
-      df
-    end
-
-    def fit(df, precomputed_stats = {})
-      return if df.nil?
-      return if preprocessing_steps.nil? || preprocessing_steps.keys.none?
-
-      preprocessing_steps.deep_symbolize_keys!
-      df = apply_clip(df, preprocessing_steps)
-
-      self.statistics = StatisticsLearner.learn_df(df, dataset: dataset, type: :raw).deep_symbolize_keys.merge!(
-        precomputed_stats
-      ).deep_symbolize_keys
-    end
-
-    def postprocess(df, inference: false, computed: false)
-      puts "Postprocessing..." if verbose
-      return df if preprocessing_steps.nil? || preprocessing_steps.keys.none?
-
-      steps = if inference
-          preprocessing_steps[:training].merge(preprocessing_steps[:inference] || {})
-        else
-          preprocessing_steps[:training]
-        end
-
-      if computed
-        computed_cols = dataset.columns.computed.map(&:name).map(&:to_sym)
-        steps = steps.deep_dup.slice(*computed_cols)
-      end
-
-      df = apply_transformations(df, steps)
-
-      puts "Postprocessing complete." if @verbose
-      df
-    end
-
-    def decode_labels(values, col: nil)
-      decoder = statistics.dig(col.to_sym, :label_decoder)
-      other_value = decoder.keys.map(&:to_s).map(&:to_i).max + 1
-      decoder[other_value] = "other"
-      decoder.stringify_keys!
-
-      values.map do |value|
-        decoder[value.to_s]
-      end
-    end
-
-    def is_fit?
-      statistics.any? { |_, col_stats| col_stats.any? { |_, strategy_stats| strategy_stats.present? } }
-    end
-
-    def delete
-      return unless File.directory?(@directory)
-
-      FileUtils.rm_rf(@directory)
-    end
-
-    def serialize
-      {
-        directory: directory,
-        verbose: verbose,
-        imputers: imputers,
-        preprocessing_steps: preprocessing_steps,
-        statistics: serialize_statistics(statistics || {}),
-      }
-    end
-
-    private
-
-    def initialize_imputers(config)
-      config.each_with_object({}) do |(col, conf), hash|
-        hash[col] ||= {}
-        conf.symbolize_keys!
-        method = conf[:method]
-        params = conf[:params] || {}
-
-        hash[col][method] = EasyML::Data::SimpleImputer.new(
-          strategy: method,
-          options: params,
-          path: directory,
-          attribute: col,
-          statistics: statistics.dig(col),
-        )
-      end
-    end
-
-    def apply_transformations(df, config)
-      imputers = initialize_imputers(config)
-
-      df = apply_clip(df, { training: config })
-
-      config.each do |col, conf|
-        conf.symbolize_keys!
-        if df.columns.map(&:downcase).map(&:to_s).include?(col.downcase.to_s)
-          actual_col = df.columns.map(&:to_s).find { |c| c.to_s.downcase == col.to_s.downcase }
-
-          strategy = conf[:method]
-          params = conf[:params]
-          imputer = imputers.dig(col, strategy)
-
-          df[actual_col] = imputer.transform(df[actual_col]) if imputer
-
-          if params.is_a?(Hash) && params.key?(:one_hot) && params[:one_hot] == true
-            df = apply_one_hot(df, col)
-          elsif params.is_a?(Hash) && params.key?(:ordinal_encoding) && params[:ordinal_encoding] == true
-            df = apply_ordinal_encoding(df, col)
-          end
-        elsif @verbose
-          puts "Warning: Column '#{col}' not found in DataFrame during apply_transformations process."
-        end
-      end
-
-      df
-    end
-
-    def apply_one_hot(df, col)
-      approved_values = statistics.dig(col, :allowed_categories).sort
-
-      # Create one-hot encoded columns
-      approved_values.each do |value|
-        new_col_name = "#{col}_#{value}".gsub(/-/, "_")
-        df = df.with_column(
-          df[col].cast(Polars::String).eq(value.to_s).cast(Polars::Boolean).alias(new_col_name)
-        )
-      end
-
-      # Create 'other' column for unapproved values
-      other_col_name = "#{col}_other"
-      df[other_col_name] = df[col].map_elements do |value|
-        approved_values.map(&:to_s).exclude?(value)
-      end.cast(Polars::Boolean)
-      df.drop([col.to_s])
-    end
-
-    def apply_ordinal_encoding(df, col)
-      approved_values = statistics.dig(col, :allowed_categories)
-
-      df.with_column(
-        df[col].map_elements do |value|
-          approved_values.map(&:to_s).exclude?(value) ? "other" : value
-        end.alias(col.to_s)
-      )
-
-      label_encoder = statistics.dig(col, :label_encoder).stringify_keys
-      other_value = label_encoder.values.max + 1
-      label_encoder["other"] = other_value
-      df.with_column(
-        df[col].map { |v| label_encoder[v.to_s] }.alias(col.to_s)
-      )
-    end
-
-    def prepare_for_imputation(df, col)
-      df = df.with_column(Polars.col(col).cast(Polars::Float64))
-      df.with_column(Polars.when(Polars.col(col).is_null).then(Float::NAN).otherwise(Polars.col(col)).alias(col))
-    end
-
-    def serialize_statistics(stats)
-      stats.deep_transform_values do |value|
-        case value
-        when Time, DateTime
-          { "__type__" => "datetime", "value" => value.iso8601 }
-        when Date
-          { "__type__" => "date", "value" => value.iso8601 }
-        when BigDecimal
-          { "__type__" => "bigdecimal", "value" => value.to_s }
-        when Polars::DataType
-          { "__type__" => "polars_dtype", "value" => value.to_s }
-        when Symbol
-          { "__type__" => "symbol", "value" => value.to_s }
-        else
-          value
-        end
-      end
-    end
-
-    def deserialize_statistics(stats)
-      return nil if stats.nil?
-
-      stats.transform_values do |value|
-        recursive_deserialize(value)
-      end
-    end
-
-    def recursive_deserialize(value)
-      case value
-      when Hash
-        if value["__type__"]
-          deserialize_special_type(value)
-        else
-          value.transform_values { |v| recursive_deserialize(v) }
-        end
-      when Array
-        value.map { |v| recursive_deserialize(v) }
-      else
-        value
-      end
-    end
-
-    def deserialize_special_type(value)
-      case value["__type__"]
-      when "datetime"
-        DateTime.parse(value["value"])
-      when "date"
-        Date.parse(value["value"])
-      when "bigdecimal"
-        BigDecimal(value["value"])
-      when "polars_dtype"
-        parse_polars_dtype(value["value"])
-      when "symbol"
-        value["value"].to_sym
-      else
-        value["value"]
-      end
-    end
-
-    def parse_polars_dtype(dtype_string)
-      case dtype_string
-      when /^Polars::Datetime/
-        time_unit = dtype_string[/time_unit: "(.*?)"/, 1]
-        time_zone = dtype_string[/time_zone: (.*)?\)/, 1]
-        time_zone = time_zone == "nil" ? nil : time_zone&.delete('"')
-        Polars::Datetime.new(time_unit: time_unit, time_zone: time_zone).class
-      when /^Polars::/
-        Polars.const_get(dtype_string.split("::").last)
-      else
-        raise ArgumentError, "Unknown Polars data type: #{dtype_string}"
-      end
-    end
-
-    def cast_to_dtype(value, dtype)
-      case dtype
-      when Polars::Int64
-        value.to_i
-      when Polars::Float64
-        value.to_f
-      when Polars::Boolean
-        !!value
-      when Polars::Utf8
-        value.to_s
-      else
-        value
-      end
-    end
-
-    def self.constants
-      {
-        preprocessing_strategies: PREPROCESSING_STRATEGIES,
-      }
-    end
-  end
-end