easy_ml 0.2.0.pre.rc76 → 0.2.0.pre.rc77

data/lib/easy_ml/feature_store.rb

@@ -1,67 +1,23 @@
 module EasyML
-  class FeatureStore
+  class FeatureStore < EasyML::Data::DatasetManager
     attr_reader :feature
 
     def initialize(feature)
       @feature = feature
-    end
-
-    def store(df)
-      primary_key = feature.primary_key&.first
-      return store_without_partitioning(df) unless df.columns.include?(primary_key)
-      return store_without_partitioning(df) unless primary_key
-
-      min_key = df[primary_key].min
-      max_key = df[primary_key].max
-      batch_size = feature.batch_size || 10_000
-
-      begin
-        # We are intentionally not using to_i, so it will raise an error for keys like "A1"
-        min_key = Integer(min_key) if min_key.is_a?(String)
-        max_key = Integer(max_key) if max_key.is_a?(String)
-      rescue ArgumentError
-        return store_without_partitioning(df)
-      end
-
-      # Only partition if we have integer keys where we can predict boundaries
-      return store_without_partitioning(df) unless min_key.is_a?(Integer) && max_key.is_a?(Integer)
-
-      partitions = compute_partition_boundaries(min_key, max_key, batch_size)
-      partitions.each do |partition_start|
-        partition_end = partition_start + batch_size - 1
-        partition_df = df.filter(
-          (Polars.col(primary_key) >= partition_start) &
-          (Polars.col(primary_key) <= partition_end)
-        )
-
-        next if partition_df.height == 0
-
-        store_partition(partition_df, primary_key, partition_start)
-      end
-    end
-
-    def query(**kwargs)
-      query_all_partitions(**kwargs)
-    end
-
-    def empty?
-      list_partitions.empty?
-    end
-
-    def list_partitions
-      Dir.glob(File.join(feature_dir, "feature*.parquet")).sort
-    end
-
-    def wipe
-      FileUtils.rm_rf(feature_dir)
-    end
 
-    def upload_remote_files
-      synced_directory.upload
-    end
+      datasource_config = feature.dataset.datasource.configuration || {}
 
-    def download
-      synced_directory&.download
+      options = {
+        root_dir: feature_dir,
+        filenames: "feature",
+        append_only: false,
+        primary_key: feature.primary_key&.first,
+        partition_size: batch_size,
+        s3_bucket: datasource_config.dig("s3_bucket") || EasyML::Configuration.s3_bucket,
+        s3_prefix: s3_prefix,
+        polars_args: datasource_config.dig("polars_args"),
+      }.compact
+      super(options)
     end
 
     def cp(old_version, new_version)
@@ -82,68 +38,8 @@ module EasyML
 
     private
 
-    def cleanup(type: :partitions)
-      case type
-      when :partitions
-        list_partitions.each do |partition|
-          FileUtils.rm(partition)
-        end
-      when :no_partitions
-        FileUtils.rm_rf(feature_path)
-      when :all
-        wipe
-      end
-    end
-
-    def store_without_partitioning(df)
-      lock_file do
-        cleanup(type: :partitions)
-        path = feature_path
-        safe_write(df, path)
-      end
-    end
-
-    def safe_write(df, path)
-      FileUtils.mkdir_p(File.dirname(path))
-      df.write_parquet(path)
-    end
-
-    def store_partition(partition_df, primary_key, partition_start)
-      lock_partition(partition_start) do
-        cleanup(type: :no_partitions)
-        path = partition_path(partition_start)
-
-        if File.exist?(path)
-          reader = EasyML::Data::PolarsReader.new
-          existing_df = reader.query([path])
-          preserved_records = existing_df.filter(
-            Polars.col(primary_key).is_in(partition_df[primary_key]).is_not
-          )
-          if preserved_records.shape[1] != partition_df.shape[1]
-            wipe
-          else
-            partition_df = Polars.concat([preserved_records, partition_df], how: "vertical")
-          end
-        end
-
-        safe_write(partition_df, path)
-      end
-    end
-
-    def query_all_partitions(**kwargs)
-      reader = EasyML::Data::PolarsReader.new
-      pattern = File.join(feature_dir, "feature*.parquet")
-      files = Dir.glob(pattern)
-
-      return Polars::DataFrame.new if files.empty?
-
-      reader.query(files, **kwargs)
-    end
-
-    def compute_partition_boundaries(min_key, max_key, batch_size)
-      start_partition = (min_key / batch_size.to_f).floor * batch_size
-      end_partition = (max_key / batch_size.to_f).floor * batch_size
-      (start_partition..end_partition).step(batch_size).to_a
+    def batch_size
+      @batch_size ||= feature.batch_size || 10_000
     end
 
     def feature_dir_for_version(version)
@@ -161,74 +57,8 @@ module EasyML
       feature_dir_for_version(feature.version)
     end
 
-    def feature_path
-      File.join(feature_dir, "feature.parquet")
-    end
-
-    def partition_path(partition_start)
-      File.join(feature_dir, "feature#{partition_start}.parquet")
-    end
-
     def s3_prefix
       File.join("datasets", feature_dir.split("datasets").last)
     end
-
-    def synced_directory
-      return unless feature.dataset&.datasource.present?
-
-      datasource_config = feature.dataset.datasource.configuration || {}
-      @synced_dir ||= EasyML::Data::SyncedDirectory.new(
-        root_dir: feature_dir,
-        s3_bucket: datasource_config.dig("s3_bucket") || EasyML::Configuration.s3_bucket,
-        s3_prefix: s3_prefix,
-        s3_access_key_id: EasyML::Configuration.s3_access_key_id,
-        s3_secret_access_key: EasyML::Configuration.s3_secret_access_key,
-        polars_args: datasource_config.dig("polars_args"),
-        cache_for: 0,
-      )
-    end
-
-    def lock_partition(partition_start)
-      Support::Lockable.with_lock(partition_lock_key(partition_start), wait_timeout: 2, stale_timeout: 60) do |client|
-        begin
-          yield client if block_given?
-        ensure
-          unlock_partition(partition_start)
-        end
-      end
-    end
-
-    def lock_file
-      Support::Lockable.with_lock(file_lock_key, wait_timeout: 2, stale_timeout: 60) do |client|
-        begin
-          yield client if block_given?
-        ensure
-          unlock_file
-        end
-      end
-    end
-
-    def unlock_partition(partition_start)
-      Support::Lockable.unlock!(partition_lock_key(partition_start))
-    end
-
-    def unlock_file
-      Support::Lockable.unlock!(file_lock_key)
-    end
-
-    def unlock_all_partitions
-      list_partitions.each do |partition_path|
-        partition_start = partition_path.match(/feature(\d+)\.parquet/)[1].to_i
-        unlock_partition(partition_start)
-      end
-    end
-
-    def partition_lock_key(partition_start)
-      "feature_store:#{feature.id}.partition.#{partition_start}"
-    end
-
-    def file_lock_key
-      "feature_store:#{feature.id}.file"
-    end
   end
 end

data/lib/easy_ml/reasons.rb

@@ -0,0 +1,41 @@
+module EasyML
+  class Reasons
+    def initialize(context)
+      @context = context
+      @reasons = {}
+    end
+
+    class << self
+      def add_reason(name, check)
+        @reasons ||= {}
+        key = name.to_s.downcase.gsub(/\s/, "_").to_sym
+        @reasons[key] = { name: name, check: check }
+      end
+
+      def reasons
+        @reasons ||= {}
+      end
+    end
+
+    def inspect
+      "#<#{self.class.name.split("::").last} checks=[#{self.class.reasons.map { |k, v| "#{v[:name]}" }.join(", ")}]>"
+    end
+
+    def none?(except: [])
+      check(except: except).none?
+    end
+
+    def check(except: [])
+      self.class.reasons.except(*except).select do |_, config|
+        @context.instance_exec(&config[:check])
+      end.map do |_, config|
+        config[:name]
+      end
+    end
+
+    def explain
+      reasons = check
+      reasons.any? ? reasons.join(", ") : :none
+    end
+  end
+end

data/lib/easy_ml/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module EasyML
-  VERSION = "0.2.0-rc76"
+  VERSION = "0.2.0-rc77"
 
   module Version
   end

data/lib/easy_ml.rb

@@ -15,13 +15,13 @@ module EasyML
   class Error < StandardError; end
 
   require_relative "easy_ml/configuration"
+  require_relative "easy_ml/reasons"
   require_relative "easy_ml/deep_compact"
   require_relative "easy_ml/timing"
   require_relative "easy_ml/support"
   require_relative "easy_ml/core_ext"
   require_relative "easy_ml/logging"
   require_relative "easy_ml/data"
-  require_relative "easy_ml/data/filter_extensions"
   require_relative "easy_ml/evaluators"
   require_relative "easy_ml/features"
   require_relative "easy_ml/feature_store"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: easy_ml
 version: !ruby/object:Gem::Version
-  version: 0.2.0.pre.rc76
+  version: 0.2.0.pre.rc77
 platform: ruby
 authors:
 - Brett Shollenberger
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-02-13 00:00:00.000000000 Z
+date: 2025-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -604,6 +604,7 @@ files:
 - app/models/easy_ml/dataset/learner/lazy/query.rb
 - app/models/easy_ml/dataset/learner/lazy/string.rb
 - app/models/easy_ml/dataset/learner/query.rb
+- app/models/easy_ml/dataset/refresh_reasons.rb
 - app/models/easy_ml/dataset_history.rb
 - app/models/easy_ml/datasource.rb
 - app/models/easy_ml/datasource_history.rb
@@ -631,6 +632,7 @@ files:
 - app/models/easy_ml/import/retraining_job.rb
 - app/models/easy_ml/import/splitter.rb
 - app/models/easy_ml/lineage.rb
+- app/models/easy_ml/lineage_history.rb
 - app/models/easy_ml/model.rb
 - app/models/easy_ml/model_file.rb
 - app/models/easy_ml/model_file_history.rb
@@ -657,7 +659,6 @@ files:
 - app/models/easy_ml/splitters/random_splitter.rb
 - app/models/easy_ml/tuner_job.rb
 - app/models/easy_ml/tuner_run.rb
-- app/models/lineage_history.rb
 - app/serializers/easy_ml/column_serializer.rb
 - app/serializers/easy_ml/dataset_serializer.rb
 - app/serializers/easy_ml/datasource_serializer.rb
@@ -705,8 +706,22 @@ files:
 - lib/easy_ml/core_ext/hash.rb
 - lib/easy_ml/core_ext/pathname.rb
 - lib/easy_ml/data.rb
+- lib/easy_ml/data/dataset_manager.rb
+- lib/easy_ml/data/dataset_manager/normalizer.rb
+- lib/easy_ml/data/dataset_manager/reader.rb
+- lib/easy_ml/data/dataset_manager/reader/base.rb
+- lib/easy_ml/data/dataset_manager/reader/batch.rb
+- lib/easy_ml/data/dataset_manager/reader/data_frame.rb
+- lib/easy_ml/data/dataset_manager/reader/file.rb
+- lib/easy_ml/data/dataset_manager/writer.rb
+- lib/easy_ml/data/dataset_manager/writer/append_only.rb
+- lib/easy_ml/data/dataset_manager/writer/base.rb
+- lib/easy_ml/data/dataset_manager/writer/named.rb
+- lib/easy_ml/data/dataset_manager/writer/partitioned.rb
+- lib/easy_ml/data/dataset_manager/writer/partitioned/partition_reasons.rb
 - lib/easy_ml/data/date_converter.rb
-- lib/easy_ml/data/filter_extensions.rb
+- lib/easy_ml/data/partition.rb
+- lib/easy_ml/data/partition/boundaries.rb
 - lib/easy_ml/data/polars_column.rb
 - lib/easy_ml/data/polars_in_memory.rb
 - lib/easy_ml/data/polars_reader.rb
@@ -765,6 +780,7 @@ files:
 - lib/easy_ml/railtie/templates/migration/remove_evaluator_from_retraining_jobs.rb.tt
 - lib/easy_ml/railtie/templates/migration/remove_preprocessor_statistics_from_easy_ml_datasets.rb.tt
 - lib/easy_ml/railtie/templates/migration/update_preprocessing_steps_to_jsonb.rb.tt
+- lib/easy_ml/reasons.rb
 - lib/easy_ml/support.rb
 - lib/easy_ml/support/age.rb
 - lib/easy_ml/support/est.rb

data/lib/easy_ml/data/filter_extensions.rb

@@ -1,31 +0,0 @@
-module EasyML
-  module Data
-    module FilterExtensions
-      def is_primary_key_filter?(primary_key)
-        return false unless primary_key
-        primary_key = [primary_key] unless primary_key.is_a?(Array)
-        # Filter expressions in Polars are represented as strings like:
-        # [([(col("LOAN_APP_ID")) > (dyn int: 4)]) & ([(col("LOAN_APP_ID")) < (dyn int: 16)])]
-        expr_str = to_s
-        return false unless expr_str.include?(primary_key.first)
-
-        # Check for common primary key operations
-        primary_key_ops = [">", "<", ">=", "<=", "=", "eq", "gt", "lt", "ge", "le"]
-        primary_key_ops.any? { |op| expr_str.include?(op) }
-      end
-
-      def extract_primary_key_values
-        expr_str = to_s
-        # Extract numeric values from the expression
-        # This will match both integers and floats
-        values = expr_str.scan(/(?:dyn int|float): (-?\d+(?:\.\d+)?)/).flatten.map(&:to_f)
-        values.uniq
-      end
-    end
-  end
-end
-
-# Extend Polars classes with our filter functionality
-[Polars::Expr].each do |klass|
-  klass.include(EasyML::Data::FilterExtensions)
-end