easy_ml 0.1.4 → 0.2.0.pre.rc1

data/lib/easy_ml/data/datasource/file_datasource.rb

@@ -1,40 +0,0 @@
-module EasyML::Data
-  class Datasource
-    class FileDatasource < Datasource
-      include GlueGun::DSL
-
-      attribute :root_dir, :string
-      attribute :polars_args, :hash, default: {}
-
-      validates :root_dir, presence: true
-
-      def in_batches(of: 10_000)
-        files.each do |file|
-          df = Polars.read_csv(file, **polars_args)
-          yield df
-        end
-      end
-
-      def files
-        Dir.glob(File.join(root_dir, "**/*.csv")).sort
-      end
-
-      def last_updated_at
-        files.map { |file| File.mtime(file) }.max
-      end
-
-      def refresh!
-        # No need to refresh for directory-based datasource
-      end
-
-      def data
-        combined_df = nil
-        files.each do |file|
-          df = Polars.read_csv(file, **polars_args)
-          combined_df = combined_df.nil? ? df : combined_df.vstack(df)
-        end
-        combined_df
-      end
-    end
-  end
-end

data/lib/easy_ml/data/datasource/merged_datasource.rb

@@ -1,64 +0,0 @@
-module EasyML::Data
-  class Datasource
-    class MergedDatasource < Datasource
-      include GlueGun::DSL
-
-      attribute :root_dir, :string
-      attribute :polars_args, :hash, default: {}
-      attribute :merge
-      validates :root_dir, presence: true
-      validates :merge, presence: true
-
-      def in_batches(of: 10_000, &block)
-        Polars.read_csv(file_path, **polars_args).iter_batches(batch_size: of, &block)
-      end
-
-      def file_path
-        @file_path ||= File.join(root_dir, "merged_data.csv")
-      end
-
-      def last_updated_at
-        datasources.map(&:last_updated_at).min
-      end
-
-      def refresh!
-        cleanup
-        if datasources.is_a?(Array)
-          datasources.each(&:refresh!)
-        elsif datasources.is_a?(Hash)
-          datasources.values.each(&:refresh!)
-        end
-      end
-
-      def data
-        @data ||= if file_exists?
-                    Polars.read_csv(file_path, **polars_args)
-                  else
-                    merge_and_save
-                  end
-      end
-
-      def cleanup
-        FileUtils.rm_f(file_path)
-      end
-
-      private
-
-      def file_exists?
-        File.exist?(file_path)
-      end
-
-      def merge_and_save
-        refresh!
-        merge.call(datasources).tap do |merged_data|
-          save_to_file(merged_data)
-        end
-      end
-
-      def save_to_file(df)
-        FileUtils.mkdir_p(root_dir)
-        df.write_csv(file_path)
-      end
-    end
-  end
-end

data/lib/easy_ml/data/datasource/polars_datasource.rb

@@ -1,41 +0,0 @@
-module EasyML::Data
-  class Datasource
-    class PolarsDatasource < Datasource
-      include GlueGun::DSL
-
-      attribute :df
-      validate :df_is_dataframe
-      def df_is_dataframe
-        return if df.nil? || df.is_a?(Polars::DataFrame)
-
-        errors.add(:df, "Must be an instance of Polars::DataFrame")
-      end
-      attr_accessor :last_updated_at
-
-      def initialize(options)
-        super
-        @last_updated_at = Time.now
-      end
-
-      def in_batches(of: 10_000)
-        total_rows = df.shape[0]
-        (0...total_rows).step(of) do |start|
-          end_index = [start + of, total_rows].min
-          yield df.slice(start, end_index - start)
-        end
-      end
-
-      def files
-        [] # No files, as this is in-memory
-      end
-
-      def refresh!
-        # No need to refresh for in-memory datasource
-      end
-
-      def data
-        df
-      end
-    end
-  end
-end

data/lib/easy_ml/data/datasource/s3_datasource.rb

@@ -1,89 +0,0 @@
-require "polars"
-
-module EasyML::Data
-  class Datasource
-    class S3Datasource
-      include GlueGun::DSL
-
-      attribute :root_dir, :string
-      validates :root_dir, presence: true
-
-      attribute :polars_args, :hash, default: {}
-      validates :polars_args, presence: true
-
-      attribute :s3_bucket, :string
-      validates :s3_bucket, presence: true
-
-      attribute :s3_prefix, :string
-      validates :s3_prefix, presence: true
-      def s3_prefix=(arg)
-        super(arg.to_s.gsub(%r{^/|/$}, ""))
-      end
-
-      attribute :s3_access_key_id, :string
-      validates :s3_access_key_id, presence: true
-
-      attribute :s3_secret_access_key, :string
-      validates :s3_secret_access_key, presence: true
-
-      dependency :synced_directory do |dependency|
-        dependency.set_class EasyML::Support::SyncedDirectory
-        dependency.bind_attribute :root_dir, required: true
-        dependency.bind_attribute :s3_bucket, required: true
-        dependency.bind_attribute :s3_prefix
-        dependency.bind_attribute :s3_access_key_id, required: true
-        dependency.bind_attribute :s3_secret_access_key, required: true
-      end
-
-      delegate :files, :last_updated_at, to: :synced_directory
-
-      def in_batches(of: 10_000)
-        # Currently ignores batch size, TODO: implement
-        pull
-        files.each do |file|
-          csv = Polars.read_csv(file, **polars_args)
-          yield csv
-        end
-      end
-
-      def refresh!
-        synced_directory.sync
-      end
-
-      def data
-        pull do |did_sync|
-          output_path = File.join(root_dir, "combined_data.csv")
-
-          if did_sync
-            combined_df = merge_data
-            combined_df.write_csv(output_path)
-          else
-            Polars.read_csv(output_path, **polars_args)
-          end
-        end
-        combined_df
-      end
-
-      private
-
-      def pull
-        # Synced directory will only sync if needs sync
-        did_sync = synced_directory.sync
-        yield did_sync if block_given?
-      end
-
-      def merge_data
-        combined_df = nil
-        files.each do |file|
-          df = Polars.read_csv(file, **polars_args)
-          combined_df = if combined_df.nil?
-                          df
-                        else
-                          combined_df.vstack(df)
-                        end
-        end
-        combined_df
-      end
-    end
-  end
-end

data/lib/easy_ml/data/datasource.rb

@@ -1,33 +0,0 @@
-module ML
-  module Data
-    class Datasource
-      attr_reader :root_dir, :polars_args
-
-      def in_batches(of: 10_000)
-        raise NotImplementedError, "Subclasses must implement #in_batches"
-      end
-
-      def files
-        raise NotImplementedError, "Subclasses must implement #files"
-      end
-
-      def last_updated_at
-        raise NotImplementedError, "Subclasses must implement #last_updated_at"
-      end
-
-      def refresh!
-        raise NotImplementedError, "Subclasses must implement #refresh!"
-      end
-
-      def data
-        raise NotImplementedError, "Subclasses must implement #data"
-      end
-
-      require_relative "datasource/s3_datasource"
-      require_relative "datasource/file_datasource"
-      require_relative "datasource/polars_datasource"
-      require_relative "datasource/merged_datasource"
-      require_relative "datasource/datasource_factory"
-    end
-  end
-end

data/lib/easy_ml/data/preprocessor/preprocessor.rb

@@ -1,205 +0,0 @@
-require "fileutils"
-require "polars"
-require "date"
-require "json"
-
-module EasyML::Data
-  class PreprocessingSteps
-    class Preprocessor
-      include EasyML::Data::PreprocessingSteps::Utils
-
-      CATEGORICAL_COMMON_MIN = 50
-      PREPROCESSING_ORDER = %w[clip mean median constant categorical one_hot ffill custom fill_date add_datepart]
-
-      attr_accessor :directory, :preprocessing_steps, :verbose, :imputers, :environment
-
-      def initialize(directory: nil, preprocessing_steps: {}, verbose: false, environment: "development")
-        @directory = directory
-        @preprocessing_steps = standardize_config(preprocessing_steps).with_indifferent_access
-        @verbose = verbose
-        @environment = environment
-      end
-
-      def fit(df)
-        return if df.nil?
-        return if preprocessing_steps.keys.none?
-
-        puts "Preprocessing..." if verbose
-        imputers = initialize_imputers(
-          preprocessing_steps[:training].merge!(preprocessing_steps[:inference] || {})
-        )
-
-        did_cleanup = false
-        imputers.each do |col, imputers|
-          sorted_strategies(imputers).each do |strategy|
-            imputer = imputers[strategy]
-            unless did_cleanup
-              imputer.cleanup
-              did_cleanup = true
-            end
-            if df.columns.map(&:downcase).include?(col.downcase)
-              actual_col = df.columns.find { |c| c.downcase == imputer.attribute.downcase }
-              imputer.fit(df[actual_col], df)
-              if strategy == "clip" # This is the only one to transform during fit
-                df[actual_col] = imputer.transform(df[actual_col])
-              end
-            elsif @verbose
-              puts "Warning: Column '#{col}' not found in DataFrame during fit process."
-            end
-          end
-        end
-      end
-
-      def postprocess(df, inference: false)
-        puts "Postprocessing..." if verbose
-        return df if preprocessing_steps.keys.none?
-
-        steps = if inference
-                  preprocessing_steps[:training].merge(preprocessing_steps[:inference] || {})
-                else
-                  preprocessing_steps[:training]
-                end
-
-        df = apply_transformations(df, steps)
-
-        puts "Postprocessing complete." if @verbose
-        df
-      end
-
-      def statistics
-        initialize_imputers(preprocessing_steps[:training]).each_with_object({}) do |(col, strategies), result|
-          result[col] = strategies.each_with_object({}) do |(strategy, imputer), col_result|
-            col_result[strategy] = imputer.statistics
-          end
-        end
-      end
-
-      def is_fit?
-        statistics.any? { |_, col_stats| col_stats.any? { |_, strategy_stats| strategy_stats.present? } }
-      end
-
-      def delete
-        return unless File.directory?(@directory)
-
-        FileUtils.rm_rf(@directory)
-      end
-
-      def move(to)
-        old_dir = directory
-        current_env = directory.split("/")[-1]
-        new_dir = directory.gsub(Regexp.new(current_env), to)
-
-        puts "Moving #{old_dir} to #{new_dir}"
-        FileUtils.mv(old_dir, new_dir)
-        @directory = new_dir
-      end
-
-      private
-
-      def initialize_imputers(config)
-        standardize_config(config).each_with_object({}) do |(col, strategies), hash|
-          hash[col] ||= {}
-          strategies.each do |strategy, options|
-            next if strategy.to_sym == :one_hot
-
-            options = {} if options == true
-
-            hash[col][strategy] = EasyML::Data::PreprocessingSteps::SimpleImputer.new(
-              strategy: strategy,
-              path: directory,
-              attribute: col,
-              options: options
-            )
-          end
-        end
-      end
-
-      def apply_transformations(df, config)
-        imputers = initialize_imputers(config)
-
-        standardize_config(config).each do |col, strategies|
-          if df.columns.map(&:downcase).include?(col.downcase)
-            actual_col = df.columns.find { |c| c.downcase == col.downcase }
-
-            sorted_strategies(strategies).each do |strategy|
-              if strategy.to_sym == :one_hot
-                df = apply_one_hot(df, col, imputers)
-              else
-                imputer = imputers.dig(col, strategy)
-                df[actual_col] = imputer.transform(df[actual_col]) if imputer
-              end
-            end
-          elsif @verbose
-            puts "Warning: Column '#{col}' not found in DataFrame during apply_transformations process."
-          end
-        end
-
-        df
-      end
-
-      def apply_one_hot(df, col, imputers)
-        approved_values = if (cat_imputer = imputers.dig(col, "categorical")).present?
-                            cat_imputer.statistics[:categorical][:value].select do |_k, v|
-                              v >= cat_imputer.options["categorical_min"]
-                            end.keys
-                          else
-                            df[col].uniq.to_a
-                          end
-
-        # Create one-hot encoded columns
-        approved_values.each do |value|
-          new_col_name = "#{col}_#{value}".gsub(/-/, "_")
-          df = df.with_column(
-            df[col].eq(value.to_s).cast(Polars::Int64).alias(new_col_name)
-          )
-        end
-
-        # Create 'other' column for unapproved values
-        other_col_name = "#{col}_other"
-        df[other_col_name] = df[col].map_elements do |value|
-          approved_values.map(&:to_s).exclude?(value)
-        end.cast(Polars::Int64)
-        df.drop([col])
-      end
-
-      def sorted_strategies(strategies)
-        strategies.keys.sort_by do |key|
-          PREPROCESSING_ORDER.index(key)
-        end
-      end
-
-      def prepare_for_imputation(df, col)
-        df = df.with_column(Polars.col(col).cast(Polars::Float64))
-        df.with_column(Polars.when(Polars.col(col).is_null).then(Float::NAN).otherwise(Polars.col(col)).alias(col))
-      end
-    end
-  end
-end
-
-# Where to put this???
-#
-# def self.stage_required_files
-#   required_files.each do |file|
-#     git_add(file)
-#   end
-# end
-
-# def self.git_add(path)
-#   command = "git add #{path}"
-#   puts command if verbose
-#   result = `#{command}`
-#   puts result if verbose
-# end
-
-# def self.set_verbose(verbose)
-#   @verbose = verbose
-# end
-
-# def required_files
-#   files = Dir.entries(@directory) - %w[. ..]
-#   required_file_types = %w[bin]
-
-#   files.select { |file| required_file_types.any? { |ext| file.include?(ext) } }.map do |file|
-#     File.join(@directory, file)
-#   end
-# end