easy_ml 0.1.1

data/lib/easy_ml/data/datasource/s3_datasource.rb

@@ -0,0 +1,89 @@
+require "polars"
+
+module EasyML::Data
+  class Datasource
+    class S3Datasource
+      include GlueGun::DSL
+
+      attribute :root_dir, :string
+      validates :root_dir, presence: true
+
+      attribute :polars_args, :hash, default: {}
+      validates :polars_args, presence: true
+
+      attribute :s3_bucket, :string
+      validates :s3_bucket, presence: true
+
+      attribute :s3_prefix, :string
+      validates :s3_prefix, presence: true
+      def s3_prefix=(arg)
+        super(arg.to_s.gsub(%r{^/|/$}, ""))
+      end
+
+      attribute :s3_access_key_id, :string
+      validates :s3_access_key_id, presence: true
+
+      attribute :s3_secret_access_key, :string
+      validates :s3_secret_access_key, presence: true
+
+      dependency :synced_directory do |dependency|
+        dependency.set_class EasyML::Support::SyncedDirectory
+        dependency.bind_attribute :root_dir, required: true
+        dependency.bind_attribute :s3_bucket, required: true
+        dependency.bind_attribute :s3_prefix
+        dependency.bind_attribute :s3_access_key_id, required: true
+        dependency.bind_attribute :s3_secret_access_key, required: true
+      end
+
+      delegate :files, :last_updated_at, to: :synced_directory
+
+      def in_batches(of: 10_000)
+        # Currently ignores batch size, TODO: implement
+        pull
+        files.each do |file|
+          csv = Polars.read_csv(file, **polars_args)
+          yield csv
+        end
+      end
+
+      def refresh!
+        synced_directory.sync
+      end
+
+      def data
+        pull do |did_sync|
+          output_path = File.join(root_dir, "combined_data.csv")
+
+          if did_sync
+            combined_df = merge_data
+            combined_df.write_csv(output_path)
+          else
+            Polars.read_csv(output_path, **polars_args)
+          end
+        end
+        combined_df
+      end
+
+      private
+
+      def pull
+        # Synced directory will only sync if needs sync
+        did_sync = synced_directory.sync
+        yield did_sync if block_given?
+      end
+
+      def merge_data
+        combined_df = nil
+        files.each do |file|
+          df = Polars.read_csv(file, **polars_args)
+          combined_df = if combined_df.nil?
+                          df
+                        else
+                          combined_df.vstack(df)
+                        end
+        end
+        combined_df
+      end
+    end
+  end
+end

data/lib/easy_ml/data/datasource.rb

@@ -0,0 +1,33 @@
+module ML
+  module Data
+    class Datasource
+      attr_reader :root_dir, :polars_args
+
+      def in_batches(of: 10_000)
+        raise NotImplementedError, "Subclasses must implement #in_batches"
+      end
+
+      def files
+        raise NotImplementedError, "Subclasses must implement #files"
+      end
+
+      def last_updated_at
+        raise NotImplementedError, "Subclasses must implement #last_updated_at"
+      end
+
+      def refresh!
+        raise NotImplementedError, "Subclasses must implement #refresh!"
+      end
+
+      def data
+        raise NotImplementedError, "Subclasses must implement #data"
+      end
+
+      require_relative "datasource/s3_datasource"
+      require_relative "datasource/file_datasource"
+      require_relative "datasource/polars_datasource"
+      require_relative "datasource/merged_datasource"
+      require_relative "datasource/datasource_factory"
+    end
+  end
+end

data/lib/easy_ml/data/preprocessor/preprocessor.rb

@@ -0,0 +1,205 @@
+require "fileutils"
+require "polars"
+require "date"
+require "json"
+
+module EasyML::Data
+  class PreprocessingSteps
+    class Preprocessor
+      include EasyML::Data::PreprocessingSteps::Utils
+
+      CATEGORICAL_COMMON_MIN = 50
+      PREPROCESSING_ORDER = %w[clip mean median constant categorical one_hot ffill custom fill_date add_datepart]
+
+      attr_accessor :directory, :preprocessing_steps, :verbose, :imputers, :environment
+
+      def initialize(directory: nil, preprocessing_steps: {}, verbose: false, environment: "development")
+        @directory = directory
+        @preprocessing_steps = standardize_config(preprocessing_steps).with_indifferent_access
+        @verbose = verbose
+        @environment = environment
+      end
+
+      def fit(df)
+        return if df.nil?
+        return if preprocessing_steps.keys.none?
+
+        puts "Preprocessing..." if verbose
+        imputers = initialize_imputers(
+          preprocessing_steps[:training].merge!(preprocessing_steps[:inference] || {})
+        )
+
+        did_cleanup = false
+        imputers.each do |col, imputers|
+          sorted_strategies(imputers).each do |strategy|
+            imputer = imputers[strategy]
+            unless did_cleanup
+              imputer.cleanup
+              did_cleanup = true
+            end
+            if df.columns.map(&:downcase).include?(col.downcase)
+              actual_col = df.columns.find { |c| c.downcase == imputer.attribute.downcase }
+              imputer.fit(df[actual_col], df)
+              if strategy == "clip" # This is the only one to transform during fit
+                df[actual_col] = imputer.transform(df[actual_col])
+              end
+            elsif @verbose
+              puts "Warning: Column '#{col}' not found in DataFrame during fit process."
+            end
+          end
+        end
+      end
+
+      def postprocess(df, inference: false)
+        puts "Postprocessing..." if verbose
+        return df if preprocessing_steps.keys.none?
+
+        steps = if inference
+                  preprocessing_steps[:training].merge(preprocessing_steps[:inference] || {})
+                else
+                  preprocessing_steps[:training]
+                end
+
+        df = apply_transformations(df, steps)
+
+        puts "Postprocessing complete." if @verbose
+        df
+      end
+
+      def statistics
+        initialize_imputers(preprocessing_steps[:training]).each_with_object({}) do |(col, strategies), result|
+          result[col] = strategies.each_with_object({}) do |(strategy, imputer), col_result|
+            col_result[strategy] = imputer.statistics
+          end
+        end
+      end
+
+      def is_fit?
+        statistics.any? { |_, col_stats| col_stats.any? { |_, strategy_stats| strategy_stats.present? } }
+      end
+
+      def delete
+        return unless File.directory?(@directory)
+
+        FileUtils.rm_rf(@directory)
+      end
+
+      def move(to)
+        old_dir = directory
+        current_env = directory.split("/")[-1]
+        new_dir = directory.gsub(Regexp.new(current_env), to)
+
+        puts "Moving #{old_dir} to #{new_dir}"
+        FileUtils.mv(old_dir, new_dir)
+        @directory = new_dir
+      end
+
+      private
+
+      def initialize_imputers(config)
+        standardize_config(config).each_with_object({}) do |(col, strategies), hash|
+          hash[col] ||= {}
+          strategies.each do |strategy, options|
+            next if strategy.to_sym == :one_hot
+
+            options = {} if options == true
+
+            hash[col][strategy] = EasyML::Data::PreprocessingSteps::SimpleImputer.new(
+              strategy: strategy,
+              path: directory,
+              attribute: col,
+              options: options
+            )
+          end
+        end
+      end
+
+      def apply_transformations(df, config)
+        imputers = initialize_imputers(config)
+
+        standardize_config(config).each do |col, strategies|
+          if df.columns.map(&:downcase).include?(col.downcase)
+            actual_col = df.columns.find { |c| c.downcase == col.downcase }
+
+            sorted_strategies(strategies).each do |strategy|
+              if strategy.to_sym == :one_hot
+                df = apply_one_hot(df, col, imputers)
+              else
+                imputer = imputers.dig(col, strategy)
+                df[actual_col] = imputer.transform(df[actual_col]) if imputer
+              end
+            end
+          elsif @verbose
+            puts "Warning: Column '#{col}' not found in DataFrame during apply_transformations process."
+          end
+        end
+
+        df
+      end
+
+      def apply_one_hot(df, col, imputers)
+        approved_values = if (cat_imputer = imputers.dig(col, "categorical")).present?
+                            cat_imputer.statistics[:categorical][:value].select do |_k, v|
+                              v >= cat_imputer.options["categorical_min"]
+                            end.keys
+                          else
+                            df[col].uniq.to_a
+                          end
+
+        # Create one-hot encoded columns
+        approved_values.each do |value|
+          new_col_name = "#{col}_#{value}".gsub(/-/, "_")
+          df = df.with_column(
+            df[col].eq(value.to_s).cast(Polars::Int64).alias(new_col_name)
+          )
+        end
+
+        # Create 'other' column for unapproved values
+        other_col_name = "#{col}_other"
+        df[other_col_name] = df[col].map_elements do |value|
+          approved_values.map(&:to_s).exclude?(value)
+        end.cast(Polars::Int64)
+        df.drop([col])
+      end
+
+      def sorted_strategies(strategies)
+        strategies.keys.sort_by do |key|
+          PREPROCESSING_ORDER.index(key)
+        end
+      end
+
+      def prepare_for_imputation(df, col)
+        df = df.with_column(Polars.col(col).cast(Polars::Float64))
+        df.with_column(Polars.when(Polars.col(col).is_null).then(Float::NAN).otherwise(Polars.col(col)).alias(col))
+      end
+    end
+  end
+end
+
+# Where to put this???
+#
+# def self.stage_required_files
+#   required_files.each do |file|
+#     git_add(file)
+#   end
+# end
+
+# def self.git_add(path)
+#   command = "git add #{path}"
+#   puts command if verbose
+#   result = `#{command}`
+#   puts result if verbose
+# end
+
+# def self.set_verbose(verbose)
+#   @verbose = verbose
+# end
+
+# def required_files
+#   files = Dir.entries(@directory) - %w[. ..]
+#   required_file_types = %w[bin]
+
+#   files.select { |file| required_file_types.any? { |ext| file.include?(ext) } }.map do |file|
+#     File.join(@directory, file)
+#   end
+# end