easy_ml 0.1.1

data/lib/easy_ml/data/preprocessor/simple_imputer.rb

@@ -0,0 +1,403 @@
+require "active_support/core_ext/hash/deep_transform_values"
+require "numo/narray"
+require "json"
+
+module EasyML
+  module Data
+    class Preprocessor
+      class SimpleImputer
+        attr_reader :statistics
+        attr_accessor :path, :attribute, :strategy, :options
+
+        def initialize(strategy: "mean", path: nil, attribute: nil, options: {}, &block)
+          @strategy = strategy.to_sym
+          @path = path
+          @attribute = attribute
+          @options = options || {}
+          apply_defaults
+          load
+          @statistics ||= {}
+          deep_symbolize_keys!
+          return unless block_given?
+
+          instance_eval(&block)
+        end
+
+        def deep_symbolize_keys!
+          @statistics = @statistics.deep_symbolize_keys
+        end
+
+        def apply_defaults
+          @options[:date_column] ||= "CREATED_DATE"
+
+          if strategy == :categorical
+            @options[:categorical_min] ||= 25
+          elsif strategy == :custom
+            itself = ->(col) { col }
+            @options[:fit] ||= itself
+            @options[:transform] ||= itself
+          end
+        end
+
+        def fit(x, df = nil)
+          x = validate_input(x)
+
+          fit_values = case @strategy
+                       when :mean
+                         fit_mean(x)
+                       when :median
+                         fit_median(x)
+                       when :ffill
+                         fit_ffill(x, df)
+                       when :most_frequent
+                         fit_most_frequent(x)
+                       when :categorical
+                         fit_categorical(x)
+                       when :constant
+                         fit_constant(x)
+                       when :clip
+                         fit_no_op(x)
+                       when :today
+                         fit_no_op(x)
+                       when :one_hot
+                         fit_no_op(x)
+                       when :custom
+                         fit_custom(x)
+                       else
+                         raise ArgumentError, "Invalid strategy: #{@strategy}"
+                       end || {}
+
+          @statistics[attribute] ||= {}
+          @statistics[attribute][@strategy] = fit_values.merge!(original_dtype: x.dtype)
+          save
+          self
+        end
+
+        def transform(x)
+          check_is_fitted
+
+          if x.is_a?(Polars::Series)
+            transform_polars(x)
+          else
+            transform_dense(x)
+          end
+        end
+
+        def transform_polars(x)
+          result = case @strategy
+                   when :mean, :median, :ffill, :most_frequent, :constant
+                     x.fill_null(@statistics[@strategy][:value])
+                   when :clip
+                     min = options["min"] || 0
+                     max = options["max"] || 1_000_000_000_000
+                     if x.null_count != x.len
+                       x.clip(min, max)
+                     else
+                       x
+                     end
+                   when :categorical
+                     allowed_values = @statistics.dig(:categorical, :value).select do |_k, v|
+                       v >= options[:categorical_min]
+                     end.keys.map(&:to_s)
+                     if x.null_count == x.len
+                       x.fill_null(transform_categorical(nil))
+                     else
+                       x.apply do |val|
+                         allowed_values.include?(val) ? val : transform_categorical(val)
+                       end
+                     end
+                   when :today
+                     x.fill_null(transform_today(nil))
+                   when :custom
+                     if x.null_count == x.len
+                       x.fill_null(transform_custom(nil))
+                     else
+                       x.apply do |val|
+                         should_transform_custom?(val) ? transform_custom(val) : val
+                       end
+                     end
+                   else
+                     raise ArgumentError, "Unsupported strategy for Polars::Series: #{@strategy}"
+                   end
+
+          # Cast the result back to the original dtype
+          original_dtype = @statistics.dig(@strategy, :original_dtype)
+          original_dtype ? result.cast(original_dtype) : result
+        end
+
+        def file_path
+          raise "Need both attribute and path to save/load statistics" unless attribute.present? && path.to_s.present?
+
+          File.join(path, "statistics.json")
+        end
+
+        def cleanup
+          @statistics = {}
+          FileUtils.rm(file_path) if File.exist?(file_path)
+        end
+
+        def save
+          FileUtils.mkdir_p(File.dirname(file_path))
+
+          all_statistics = (File.exist?(file_path) ? JSON.parse(File.read(file_path)) : {}).deep_symbolize_keys
+
+          deep_symbolize_keys!
+
+          serialized = serialize_statistics(@statistics)
+          all_statistics[attribute] = {} unless all_statistics.key?(attribute)
+          all_statistics[attribute][@strategy] = serialized[attribute.to_sym][@strategy.to_sym]
+
+          File.open(file_path, "w") do |file|
+            file.write(JSON.pretty_generate(all_statistics))
+          end
+        end
+
+        def load
+          return unless File.exist?(file_path)
+
+          all_statistics = JSON.parse(File.read(file_path))
+          attribute_stats = all_statistics[@attribute]
+
+          return unless attribute_stats
+
+          @statistics = deserialize_statistics(attribute_stats)
+          deep_symbolize_keys!
+        end
+
+        def should_transform_categorical?(val)
+          values = @statistics.dig(:categorical, :value) || {}
+          min_ct = options[:categorical_min] || 25
+          allowed_values = values.select { |_v, c| c >= min_ct }
+
+          allowed_values.keys.map(&:to_s).exclude?(val)
+        end
+
+        def transform_categorical(val)
+          return "other" if val.nil?
+
+          values = @statistics.dig(:categorical, :value) || {}
+          min_ct = options[:categorical_min] || 25
+          allowed_values = values.select { |_v, c| c >= min_ct }.keys.map(&:to_s)
+
+          binding.pry
+          allowed_values.include?(val.to_s) ? val.to_s : "other"
+        end
+
+        def transform_today(_val)
+          EST.now.beginning_of_day
+        end
+
+        def fit_custom(x)
+          x
+        end
+
+        def should_transform_custom?(x)
+          if options.key?(:should_transform)
+            options[:should_transform].call(x)
+          else
+            should_transform_default?(x)
+          end
+        end
+
+        def transform_custom(x)
+          raise "Transform required" unless options.key?(:transform)
+
+          options[:transform].call(x)
+        end
+
+        private
+
+        def validate_input(x)
+          raise ArgumentError, "Input must be a Polars::Series" unless x.is_a?(Polars::Series)
+
+          x
+        end
+
+        def fit_mean(x)
+          { value: x.mean }
+        end
+
+        def fit_median(x)
+          { value: x.median }
+        end
+
+        def fit_ffill(x, df)
+          values = { value: nil, max_date: nil }
+
+          date_col = df[options[:date_column]]
+          return if date_col.is_null.all
+
+          sorted_df = df.sort(options[:date_column])
+          new_max_date = sorted_df[options[:date_column]].max
+
+          current_max_date = values[:max_date]
+          return if current_max_date && current_max_date > new_max_date
+
+          values[:max_date] = [current_max_date, new_max_date].compact.max
+
+          # Get the last non-null value
+          last_non_null = sorted_df[x.name].filter(sorted_df[x.name].is_not_null).tail(1).to_a.first
+          values[:value] = last_non_null
+
+          values
+        end
+
+        def fit_most_frequent(x)
+          value_counts = x.filter(x.is_not_null).value_counts
+          column_names = value_counts.columns
+          column_names[0]
+          count_column = column_names[1]
+
+          most_frequent_value = value_counts.sort(count_column, descending: true).row(0)[0]
+          { value: most_frequent_value }
+        end
+
+        def fit_categorical(x)
+          value_counts = x.value_counts
+          column_names = value_counts.columns
+          value_column = column_names[0]
+          count_column = column_names[1]
+
+          as_hash = value_counts.select([value_column, count_column]).rows.to_a.to_h.transform_keys(&:to_s)
+          label_encoder = as_hash.keys.sort.each.with_index.reduce({}) do |h, (k, i)|
+            h.tap do
+              h[k] = i
+            end
+          end
+          label_decoder = label_encoder.invert
+
+          {
+            value: as_hash,
+            label_encoder: label_encoder,
+            label_decoder: label_decoder
+          }
+        end
+
+        def fit_no_op(_x)
+          {}
+        end
+
+        def fit_constant(_x)
+          { value: @options[:fill_value] }
+        end
+
+        def transform_default(_val)
+          @statistics[strategy][:value]
+        end
+
+        def should_transform_default?(val)
+          checker_method = val.respond_to?(:nan?) ? :nan? : :nil?
+          val.send(checker_method)
+        end
+
+        def transform_dense(x)
+          result = x.map do |val|
+            strategy_method = respond_to?("transform_#{strategy}") ? "transform_#{strategy}" : "transform_default"
+            checker_method = respond_to?("should_transform_#{strategy}?") ? "should_transform_#{strategy}?" : "should_transform_default?"
+            send(checker_method, val) ? send(strategy_method, val) : val
+          end
+
+          # Cast the result back to the original dtype
+          original_dtype = @statistics[:original_dtype]
+          if original_dtype
+            result.map { |val| cast_to_dtype(val, original_dtype) }
+          else
+            result
+          end
+        end
+
+        def check_is_fitted
+          return if %i[clip today custom].include?(strategy)
+
+          raise "SimpleImputer has not been fitted yet for #{attribute}##{strategy}" unless @statistics[strategy]
+        end
+
+        def serialize_statistics(stats)
+          stats.deep_transform_values do |value|
+            case value
+            when Time, DateTime
+              { "__type__" => "datetime", "value" => value.iso8601 }
+            when Date
+              { "__type__" => "date", "value" => value.iso8601 }
+            when BigDecimal
+              { "__type__" => "bigdecimal", "value" => value.to_s }
+            when Polars::DataType
+              { "__type__" => "polars_dtype", "value" => value.to_s }
+            when Symbol
+              { "__type__" => "symbol", "value" => value.to_s }
+            else
+              value
+            end
+          end
+        end
+
+        def deserialize_statistics(stats)
+          stats.transform_values do |value|
+            recursive_deserialize(value)
+          end
+        end
+
+        def recursive_deserialize(value)
+          case value
+          when Hash
+            if value["__type__"]
+              deserialize_special_type(value)
+            else
+              value.transform_values { |v| recursive_deserialize(v) }
+            end
+          when Array
+            value.map { |v| recursive_deserialize(v) }
+          else
+            value
+          end
+        end
+
+        def deserialize_special_type(value)
+          case value["__type__"]
+          when "datetime"
+            DateTime.parse(value["value"])
+          when "date"
+            Date.parse(value["value"])
+          when "bigdecimal"
+            BigDecimal(value["value"])
+          when "polars_dtype"
+            parse_polars_dtype(value["value"])
+          when "symbol"
+            value["value"].to_sym
+          else
+            value["value"]
+          end
+        end
+
+        def parse_polars_dtype(dtype_string)
+          case dtype_string
+          when /^Polars::Datetime/
+            time_unit = dtype_string[/time_unit: "(.*?)"/, 1]
+            time_zone = dtype_string[/time_zone: (.*)?\)/, 1]
+            time_zone = time_zone == "nil" ? nil : time_zone&.delete('"')
+            Polars::Datetime.new(time_unit: time_unit, time_zone: time_zone).class
+          when /^Polars::/
+            Polars.const_get(dtype_string.split("::").last)
+          else
+            raise ArgumentError, "Unknown Polars data type: #{dtype_string}"
+          end
+        end
+
+        def cast_to_dtype(value, dtype)
+          case dtype
+          when Polars::Int64
+            value.to_i
+          when Polars::Float64
+            value.to_f
+          when Polars::Boolean
+            !!value
+          when Polars::Utf8
+            value.to_s
+          else
+            value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/easy_ml/data/preprocessor/utils.rb

@@ -0,0 +1,17 @@
+module EasyML::Data
+  class Preprocessor
+    module Utils
+      def standardize_config(config)
+        config.each do |column, strategies|
+          next unless strategies.is_a?(Array)
+
+          config[column] = strategies.reduce({}) do |hash, strategy|
+            hash.tap do
+              hash[strategy] = true
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/easy_ml/data/preprocessor.rb

@@ -0,0 +1,238 @@
+require "fileutils"
+require "polars"
+require "date"
+require "json"
+require_relative "preprocessor/utils"
+require_relative "preprocessor/simple_imputer"
+
+module EasyML::Data
+  class Preprocessor
+    include GlueGun::DSL
+    include EasyML::Data::Preprocessor::Utils
+
+    CATEGORICAL_COMMON_MIN = 50
+    PREPROCESSING_ORDER = %w[clip mean median constant categorical one_hot ffill custom fill_date add_datepart]
+
+    attribute :directory, :string
+    attribute :verbose, :boolean, default: false
+    attribute :preprocessing_steps, :hash, default: {}
+    def preprocessing_steps=(preprocessing_steps)
+      super(standardize_config(preprocessing_steps).with_indifferent_access)
+    end
+
+    def fit(df)
+      return if df.nil?
+      return if preprocessing_steps.keys.none?
+
+      puts "Preprocessing..." if verbose
+      imputers = initialize_imputers(
+        preprocessing_steps[:training].merge!(preprocessing_steps[:inference] || {})
+      )
+
+      did_cleanup = false
+      imputers.each do |col, imputers|
+        sorted_strategies(imputers).each do |strategy|
+          imputer = imputers[strategy]
+          unless did_cleanup
+            imputer.cleanup
+            did_cleanup = true
+          end
+          if df.columns.map(&:downcase).include?(col.downcase)
+            actual_col = df.columns.find { |c| c.downcase == imputer.attribute.downcase }
+            imputer.fit(df[actual_col], df)
+            if strategy == "clip" # This is the only one to transform during fit
+              df[actual_col] = imputer.transform(df[actual_col])
+            end
+          elsif @verbose
+            puts "Warning: Column '#{col}' not found in DataFrame during fit process."
+          end
+        end
+      end
+    end
+
+    def postprocess(df, inference: false)
+      puts "Postprocessing..." if verbose
+      return df if preprocessing_steps.keys.none?
+
+      steps = if inference
+                preprocessing_steps[:training].merge(preprocessing_steps[:inference] || {})
+              else
+                preprocessing_steps[:training]
+              end
+
+      df = apply_transformations(df, steps)
+
+      puts "Postprocessing complete." if @verbose
+      df
+    end
+
+    def statistics
+      initialize_imputers(preprocessing_steps[:training]).each_with_object({}) do |(col, strategies), result|
+        result[col] = strategies.each_with_object({}) do |(strategy, imputer), col_result|
+          col_result[strategy] = imputer.statistics
+        end
+      end
+    end
+
+    def is_fit?
+      statistics.any? { |_, col_stats| col_stats.any? { |_, strategy_stats| strategy_stats.present? } }
+    end
+
+    def delete
+      return unless File.directory?(@directory)
+
+      FileUtils.rm_rf(@directory)
+    end
+
+    def move(to)
+      old_dir = directory
+      current_env = directory.split("/")[-1]
+      new_dir = directory.gsub(Regexp.new(current_env), to)
+
+      puts "Moving #{old_dir} to #{new_dir}"
+      FileUtils.mv(old_dir, new_dir)
+      @directory = new_dir
+    end
+
+    def decode_labels(values, col: nil)
+      imputers = initialize_imputers(preprocessing_steps[:training], dumb: true)
+      imputer = imputers.dig(col, "categorical")
+      decoder = imputer.statistics.dig(:categorical, :label_decoder)
+
+      other_value = decoder.keys.map(&:to_s).map(&:to_i).max + 1
+      decoder[other_value] = "other"
+      decoder.stringify_keys!
+
+      values.map do |value|
+        decoder[value.to_s]
+      end
+    end
+
+    private
+
+    def initialize_imputers(config, dumb: false)
+      standardize_config(config).each_with_object({}) do |(col, strategies), hash|
+        hash[col] ||= {}
+        strategies.each do |strategy, options|
+          options = {} if options == true
+
+          hash[col][strategy] = EasyML::Data::Preprocessor::SimpleImputer.new(
+            strategy: strategy,
+            path: directory,
+            attribute: col,
+            options: options
+          )
+        end
+      end
+    end
+
+    def apply_transformations(df, config)
+      imputers = initialize_imputers(config)
+
+      standardize_config(config).each do |col, strategies|
+        if df.columns.map(&:downcase).include?(col.downcase)
+          actual_col = df.columns.find { |c| c.downcase == col.downcase }
+
+          sorted_strategies(strategies).each do |strategy|
+            if strategy.to_sym == :categorical
+              if imputers.dig(col, strategy).options.dig("one_hot")
+                df = apply_one_hot(df, col, imputers)
+              elsif imputers.dig(col, strategy).options.dig("encode_labels")
+                df = apply_encode_labels(df, col, imputers)
+              end
+            else
+              imputer = imputers.dig(col, strategy)
+              df[actual_col] = imputer.transform(df[actual_col]) if imputer
+            end
+          end
+        elsif @verbose
+          puts "Warning: Column '#{col}' not found in DataFrame during apply_transformations process."
+        end
+      end
+
+      df
+    end
+
+    def apply_one_hot(df, col, imputers)
+      cat_imputer = imputers.dig(col, "categorical")
+      approved_values = cat_imputer.statistics[:categorical][:value].select do |_k, v|
+        v >= cat_imputer.options["categorical_min"]
+      end.keys
+
+      # Create one-hot encoded columns
+      approved_values.each do |value|
+        new_col_name = "#{col}_#{value}".tr("-", "_")
+        df = df.with_column(
+          df[col].eq(value.to_s).cast(Polars::Int64).alias(new_col_name)
+        )
+      end
+
+      # Create 'other' column for unapproved values
+      other_col_name = "#{col}_other"
+      df[other_col_name] = df[col].map_elements do |value|
+        approved_values.map(&:to_s).exclude?(value)
+      end.cast(Polars::Int64)
+      df.drop([col])
+    end
+
+    def apply_encode_labels(df, col, imputers)
+      cat_imputer = imputers.dig(col, "categorical")
+      approved_values = cat_imputer.statistics[:categorical][:value].select do |_k, v|
+        v >= cat_imputer.options["categorical_min"]
+      end.keys
+
+      df.with_column(
+        df[col].map_elements do |value|
+          approved_values.map(&:to_s).exclude?(value) ? "other" : value
+        end.alias(col)
+      )
+
+      label_encoder = cat_imputer.statistics[:categorical][:label_encoder].stringify_keys
+      other_value = label_encoder.values.max + 1
+      label_encoder["other"] = other_value
+
+      df.with_column(
+        df[col].map { |v| label_encoder[v.to_s] }.alias(col)
+      )
+    end
+
+    def sorted_strategies(strategies)
+      strategies.keys.sort_by do |key|
+        PREPROCESSING_ORDER.index(key)
+      end
+    end
+
+    def prepare_for_imputation(df, col)
+      df = df.with_column(Polars.col(col).cast(Polars::Float64))
+      df.with_column(Polars.when(Polars.col(col).is_null).then(Float::NAN).otherwise(Polars.col(col)).alias(col))
+    end
+  end
+end
+
+# Where to put this???
+#
+# def self.stage_required_files
+#   required_files.each do |file|
+#     git_add(file)
+#   end
+# end
+
+# def self.git_add(path)
+#   command = "git add #{path}"
+#   puts command if verbose
+#   result = `#{command}`
+#   puts result if verbose
+# end
+
+# def self.set_verbose(verbose)
+#   @verbose = verbose
+# end
+
+# def required_files
+#   files = Dir.entries(@directory) - %w[. ..]
+#   required_file_types = %w[bin]
+
+#   files.select { |file| required_file_types.any? { |ext| file.include?(ext) } }.map do |file|
+#     File.join(@directory, file)
+#   end
+# end

data/lib/easy_ml/data/utils.rb

@@ -0,0 +1,50 @@
+module EasyML
+  module Data
+    module Utils
+      def append_to_csv(df, path)
+        return if df.empty?
+
+        path = Pathname.new(path) if path.is_a?(String)
+        FileUtils.mkdir_p(path.dirname)
+        FileUtils.touch(path)
+
+        # Check if the file is empty (i.e., if this is the first write)
+        file_empty = File.zero?(path)
+
+        # Write the DataFrame to a temporary file
+        temp_file = "#{path}.tmp"
+        df.write_csv(temp_file)
+
+        # Append the content to the main file, skipping the header if not the first write
+        File.open(path, "a") do |f|
+          File.foreach(temp_file).with_index do |line, index|
+            # Skip the header line if the file is not empty
+            f.write(line) unless index == 0 && !file_empty
+          end
+        end
+
+        # Delete the temporary file
+        File.delete(temp_file)
+      end
+
+      def expand_dir(dir)
+        return dir if dir.to_s[0] == "/"
+
+        Rails.root.join(dir)
+      end
+
+      def null_check(df)
+        result = {}
+        null_counts = df.null_count
+        total_count = df.height
+        df.columns.each do |column|
+          null_count = null_counts[column][0]
+          next if null_count == 0
+
+          result[column] = { null_count: null_count, total_count: total_count }
+        end
+        result.empty? ? nil : result
+      end
+    end
+  end
+end