easy_ml 0.1.1

data/lib/easy_ml/core/uploaders.rb

@@ -0,0 +1,7 @@
+module EasyML
+  module Core
+    module Uploaders
+      require_relative "uploaders/model_uploader"
+    end
+  end
+end

data/lib/easy_ml/core.rb

@@ -0,0 +1,9 @@
+module EasyML
+  module Core
+    require_relative "core/uploaders"
+    require_relative "core/model"
+    require_relative "core/models"
+    require_relative "core/model_evaluator"
+    require_relative "core/tuner"
+  end
+end

data/lib/easy_ml/core_ext/pathname.rb

@@ -0,0 +1,9 @@
+require "pathname"
+
+class Pathname
+  def append(folder)
+    dir = cleanpath
+    dir = dir.join(folder) unless basename.to_s == folder
+    dir
+  end
+end

data/lib/easy_ml/core_ext.rb

@@ -0,0 +1,5 @@
+module EasyML
+  module CoreExt
+    require_relative "core_ext/pathname"
+  end
+end

data/lib/easy_ml/data/dataloader.rb

@@ -0,0 +1,6 @@
+module ML
+  module Data
+    class Dataloader
+    end
+  end
+end

data/lib/easy_ml/data/dataset/data/preprocessor/statistics.json

@@ -0,0 +1,31 @@
+{
+  "annual_revenue": {
+    "median": {
+      "value": 3000.0,
+      "original_dtype": {
+        "__type__": "polars_dtype",
+        "value": "Polars::Int64"
+      }
+    }
+  },
+  "loan_purpose": {
+    "categorical": {
+      "value": {
+        "payroll": 4,
+        "expansion": 1
+      },
+      "label_encoder": {
+        "expansion": 0,
+        "payroll": 1
+      },
+      "label_decoder": {
+        "0": "expansion",
+        "1": "payroll"
+      },
+      "original_dtype": {
+        "__type__": "polars_dtype",
+        "value": "Polars::String"
+      }
+    }
+  }
+}

data/lib/easy_ml/data/dataset/data/sample_info.json

@@ -0,0 +1 @@
+{"previous_sample":1.0}

data/lib/easy_ml/data/dataset/dataset/files/sample_info.json

@@ -0,0 +1 @@
+{"previous_sample":1.0}

data/lib/easy_ml/data/dataset/splits/file_split.rb

@@ -0,0 +1,140 @@
+require_relative "split"
+
+module EasyML
+  module Data
+    class Dataset
+      module Splits
+        class FileSplit < Split
+          include GlueGun::DSL
+          include EasyML::Data::Utils
+
+          attribute :dir, :string
+          attribute :polars_args, :hash, default: {}
+          attribute :max_rows_per_file, :integer, default: 1_000_000
+          attribute :batch_size, :integer, default: 10_000
+          attribute :sample, :float, default: 1.0
+          attribute :verbose, :boolean, default: false
+
+          def initialize(options)
+            super
+            FileUtils.mkdir_p(dir)
+          end
+
+          def save(segment, df)
+            segment_dir = File.join(dir, segment.to_s)
+            FileUtils.mkdir_p(segment_dir)
+
+            current_file = current_file_for_segment(segment)
+            current_row_count = current_file && File.exist?(current_file) ? df(current_file).shape[0] : 0
+            remaining_rows = max_rows_per_file - current_row_count
+
+            while df.shape[0] > 0
+              if df.shape[0] <= remaining_rows
+                append_to_csv(df, current_file)
+                break
+              else
+                df_to_append = df.slice(0, remaining_rows)
+                df = df.slice(remaining_rows, df.shape[0] - remaining_rows)
+                append_to_csv(df_to_append, current_file)
+                current_file = new_file_path_for_segment(segment)
+                remaining_rows = max_rows_per_file
+              end
+            end
+          end
+
+          def read(segment, split_ys: false, target: nil, drop_cols: [], &block)
+            files = files_for_segment(segment)
+
+            if block_given?
+              result = nil
+              total_rows = files.sum { |file| df(file).shape[0] }
+              progress_bar = create_progress_bar(segment, total_rows) if verbose
+
+              files.each do |file|
+                df = self.df(file)
+                df = sample_data(df) if sample < 1.0
+                drop_cols &= df.columns
+                df = df.drop(drop_cols) unless drop_cols.empty?
+
+                if split_ys
+                  xs, ys = split_features_targets(df, true, target)
+                  result = process_block_with_split_ys(block, result, xs, ys)
+                else
+                  result = process_block_without_split_ys(block, result, df)
+                end
+
+                progress_bar.progress += df.shape[0] if verbose
+              end
+              progress_bar.finish if verbose
+              result
+            elsif files.empty?
+              return nil, nil if split_ys
+
+              nil
+
+            else
+              combined_df = combine_dataframes(files)
+              combined_df = sample_data(combined_df) if sample < 1.0
+              drop_cols &= combined_df.columns
+              combined_df = combined_df.drop(drop_cols) unless drop_cols.empty?
+              split_features_targets(combined_df, split_ys, target)
+            end
+          end
+
+          def cleanup
+            FileUtils.rm_rf(dir)
+            FileUtils.mkdir_p(dir)
+          end
+
+          def split_at
+            return nil if output_files.empty?
+
+            output_files.map { |file| File.mtime(file) }.max
+          end
+
+          private
+
+          def read_csv_batched(path)
+            Polars.read_csv_batched(path, batch_size: batch_size, **polars_args)
+          end
+
+          def df(path)
+            Polars.read_csv(path, **polars_args)
+          end
+
+          def output_files
+            Dir.glob("#{dir}/**/*.csv")
+          end
+
+          def files_for_segment(segment)
+            segment_dir = File.join(dir, segment.to_s)
+            Dir.glob(File.join(segment_dir, "**/*.csv")).sort
+          end
+
+          def current_file_for_segment(segment)
+            current_file = files_for_segment(segment).last
+            return new_file_path_for_segment(segment) if current_file.nil?
+
+            row_count = df(current_file).shape[0]
+            if row_count >= max_rows_per_file
+              new_file_path_for_segment(segment)
+            else
+              current_file
+            end
+          end
+
+          def new_file_path_for_segment(segment)
+            segment_dir = File.join(dir, segment.to_s)
+            file_number = Dir.glob(File.join(segment_dir, "*.csv")).count
+            File.join(segment_dir, "#{segment}_%04d.csv" % file_number)
+          end
+
+          def combine_dataframes(files)
+            dfs = files.map { |file| df(file) }
+            Polars.concat(dfs)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/easy_ml/data/dataset/splits/in_memory_split.rb

@@ -0,0 +1,49 @@
+module EasyML
+  module Data
+    class Dataset
+      module Splits
+        class InMemorySplit < Split
+          include GlueGun::DSL
+
+          attribute :sample, :float, default: 1.0
+          def initialize(options)
+            super
+            @data = {}
+          end
+
+          def save(segment, df)
+            @data[segment] = df
+          end
+
+          def read(segment, split_ys: false, target: nil, drop_cols: [], &block)
+            df = @data[segment]
+            return nil if df.nil?
+
+            df = sample_data(df) if sample < 1.0
+            drop_cols &= df.columns
+            df = df.drop(drop_cols) unless drop_cols.empty?
+
+            if block_given?
+              if split_ys
+                xs, ys = split_features_targets(df, true, target)
+                process_block_with_split_ys(block, nil, xs, ys)
+              else
+                process_block_without_split_ys(block, nil, df)
+              end
+            else
+              split_features_targets(df, split_ys, target)
+            end
+          end
+
+          def cleanup
+            @data.clear
+          end
+
+          def split_at
+            @data.keys.empty? ? nil : Time.now
+          end
+        end
+      end
+    end
+  end
+end

data/lib/easy_ml/data/dataset/splits/split.rb

@@ -0,0 +1,98 @@
+module EasyML
+  module Data
+    class Dataset
+      module Splits
+        class Split
+          include GlueGun::DSL
+          include EasyML::Data::Utils
+
+          attribute :polars_args, :hash, default: {}
+          attribute :max_rows_per_file, :integer, default: 1_000_000
+          attribute :batch_size, :integer, default: 10_000
+          attribute :sample, :float, default: 1.0
+          attribute :verbose, :boolean, default: false
+
+          def save(segment, df)
+            raise NotImplementedError, "Subclasses must implement #save"
+          end
+
+          def read(segment, split_ys: false, target: nil, drop_cols: [], &block)
+            raise NotImplementedError, "Subclasses must implement #read"
+          end
+
+          def train(&block)
+            read(:train, &block)
+          end
+
+          def test(&block)
+            read(:test, &block)
+          end
+
+          def valid(&block)
+            read(:valid, &block)
+          end
+
+          def cleanup
+            raise NotImplementedError, "Subclasses must implement #cleanup"
+          end
+
+          def split_at
+            raise NotImplementedError, "Subclasses must implement #split_at"
+          end
+
+          protected
+
+          def split_features_targets(df, split_ys, target)
+            raise ArgumentError, "Target column must be specified when split_ys is true" if split_ys && target.nil?
+
+            if split_ys
+              xs = df.drop(target)
+              ys = df.select(target)
+              [xs, ys]
+            else
+              df
+            end
+          end
+
+          def sample_data(df)
+            return df if sample >= 1.0
+
+            df.sample(n: (df.shape[0] * sample).ceil, seed: 42)
+          end
+
+          def create_progress_bar(segment, total_rows)
+            ProgressBar.create(
+              title: "Reading #{segment}",
+              total: total_rows,
+              format: "%t: |%B| %p%% %e"
+            )
+          end
+
+          def process_block_with_split_ys(block, result, xs, ys)
+            case block.arity
+            when 3
+              result.nil? ? [xs, ys] : block.call(result, xs, ys)
+            when 2
+              block.call(xs, ys)
+              result
+            else
+              raise ArgumentError, "Block must accept 2 or 3 arguments when split_ys is true"
+            end
+          end
+
+          def process_block_without_split_ys(block, result, df)
+            case block.arity
+            when 2
+              result.nil? ? df : block.call(result, df)
+            when 1
+              block.call(df)
+              result
+            else
+              raise ArgumentError, "Block must accept 1 or 2 arguments when split_ys is false"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/easy_ml/data/dataset/splits.rb

@@ -0,0 +1,11 @@
+module EasyML
+  module Data
+    class Dataset
+      module Splits
+        require_relative "splits/split"
+        require_relative "splits/file_split"
+        require_relative "splits/in_memory_split"
+      end
+    end
+  end
+end

data/lib/easy_ml/data/dataset/splitters/date_splitter.rb

@@ -0,0 +1,43 @@
+module EasyML::Data::Dataset::Splitters
+  class DateSplitter
+    include GlueGun::DSL
+
+    attribute :today, :datetime
+    def today=(value)
+      super(value.in_time_zone(UTC).to_datetime)
+    end
+    attribute :date_col, :string
+    attribute :months_test, :integer, default: 2
+    attribute :months_valid, :integer, default: 2
+
+    def initialize(options)
+      options[:today] ||= UTC.now
+      super(options)
+    end
+
+    def split(df)
+      unless df[date_col].dtype.is_a?(Polars::Datetime)
+        raise "Date splitter cannot split on non-date col #{date_col}, dtype is #{df[date_col].dtype}"
+      end
+
+      validation_date_start, test_date_start = splits
+
+      test_df = df.filter(Polars.col(date_col) >= test_date_start)
+      remaining_df = df.filter(Polars.col(date_col) < test_date_start)
+      valid_df = remaining_df.filter(Polars.col(date_col) >= validation_date_start)
+      train_df = remaining_df.filter(Polars.col(date_col) < validation_date_start)
+
+      [train_df, valid_df, test_df]
+    end
+
+    def months(n)
+      ActiveSupport::Duration.months(n)
+    end
+
+    def splits
+      test_date_start = today.advance(months: -months_test).beginning_of_day
+      validation_date_start = today.advance(months: -(months_test + months_valid)).beginning_of_day
+      [validation_date_start, test_date_start]
+    end
+  end
+end

data/lib/easy_ml/data/dataset/splitters.rb

@@ -0,0 +1,9 @@
+module EasyML
+  module Data
+    class Dataset
+      module Splitters
+        require_relative "splitters/date_splitter"
+      end
+    end
+  end
+end