easy_ml 0.2.0.pre.rc76 → 0.2.0.pre.rc78

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: easy_ml
 version: !ruby/object:Gem::Version
-  version: 0.2.0.pre.rc76
+  version: 0.2.0.pre.rc78
 platform: ruby
 authors:
 - Brett Shollenberger
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-02-13 00:00:00.000000000 Z
+date: 2025-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -198,14 +198,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.16.0
+        version: 0.17.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.16.0
+        version: 0.17.0
 - !ruby/object:Gem::Dependency
   name: pycall
   requirement: !ruby/object:Gem::Requirement
@@ -604,12 +604,12 @@ files:
 - app/models/easy_ml/dataset/learner/lazy/query.rb
 - app/models/easy_ml/dataset/learner/lazy/string.rb
 - app/models/easy_ml/dataset/learner/query.rb
+- app/models/easy_ml/dataset/refresh_reasons.rb
 - app/models/easy_ml/dataset_history.rb
 - app/models/easy_ml/datasource.rb
 - app/models/easy_ml/datasource_history.rb
 - app/models/easy_ml/datasources/base_datasource.rb
 - app/models/easy_ml/datasources/file_datasource.rb
-- app/models/easy_ml/datasources/polars_datasource.rb
 - app/models/easy_ml/datasources/s3_datasource.rb
 - app/models/easy_ml/deploy.rb
 - app/models/easy_ml/event.rb
@@ -631,6 +631,7 @@ files:
 - app/models/easy_ml/import/retraining_job.rb
 - app/models/easy_ml/import/splitter.rb
 - app/models/easy_ml/lineage.rb
+- app/models/easy_ml/lineage_history.rb
 - app/models/easy_ml/model.rb
 - app/models/easy_ml/model_file.rb
 - app/models/easy_ml/model_file_history.rb
@@ -657,7 +658,6 @@ files:
 - app/models/easy_ml/splitters/random_splitter.rb
 - app/models/easy_ml/tuner_job.rb
 - app/models/easy_ml/tuner_run.rb
-- app/models/lineage_history.rb
 - app/serializers/easy_ml/column_serializer.rb
 - app/serializers/easy_ml/dataset_serializer.rb
 - app/serializers/easy_ml/datasource_serializer.rb
@@ -705,8 +705,22 @@ files:
 - lib/easy_ml/core_ext/hash.rb
 - lib/easy_ml/core_ext/pathname.rb
 - lib/easy_ml/data.rb
+- lib/easy_ml/data/dataset_manager.rb
+- lib/easy_ml/data/dataset_manager/normalizer.rb
+- lib/easy_ml/data/dataset_manager/reader.rb
+- lib/easy_ml/data/dataset_manager/reader/base.rb
+- lib/easy_ml/data/dataset_manager/reader/batch.rb
+- lib/easy_ml/data/dataset_manager/reader/data_frame.rb
+- lib/easy_ml/data/dataset_manager/reader/file.rb
+- lib/easy_ml/data/dataset_manager/writer.rb
+- lib/easy_ml/data/dataset_manager/writer/append_only.rb
+- lib/easy_ml/data/dataset_manager/writer/base.rb
+- lib/easy_ml/data/dataset_manager/writer/named.rb
+- lib/easy_ml/data/dataset_manager/writer/partitioned.rb
+- lib/easy_ml/data/dataset_manager/writer/partitioned/partition_reasons.rb
 - lib/easy_ml/data/date_converter.rb
-- lib/easy_ml/data/filter_extensions.rb
+- lib/easy_ml/data/partition.rb
+- lib/easy_ml/data/partition/boundaries.rb
 - lib/easy_ml/data/polars_column.rb
 - lib/easy_ml/data/polars_in_memory.rb
 - lib/easy_ml/data/polars_reader.rb
@@ -765,6 +779,7 @@ files:
 - lib/easy_ml/railtie/templates/migration/remove_evaluator_from_retraining_jobs.rb.tt
 - lib/easy_ml/railtie/templates/migration/remove_preprocessor_statistics_from_easy_ml_datasets.rb.tt
 - lib/easy_ml/railtie/templates/migration/update_preprocessing_steps_to_jsonb.rb.tt
+- lib/easy_ml/reasons.rb
 - lib/easy_ml/support.rb
 - lib/easy_ml/support/age.rb
 - lib/easy_ml/support/est.rb
@@ -784,8 +799,8 @@ files:
 - public/easy_ml/assets/.vite/manifest-assets.json
 - public/easy_ml/assets/.vite/manifest.json
 - public/easy_ml/assets/assets/Application-nnn_XLuL.css
-- public/easy_ml/assets/assets/entrypoints/Application.tsx-B1qLZuyu.js
-- public/easy_ml/assets/assets/entrypoints/Application.tsx-B1qLZuyu.js.map
+- public/easy_ml/assets/assets/entrypoints/Application.tsx-Bbf3mD_b.js
+- public/easy_ml/assets/assets/entrypoints/Application.tsx-Bbf3mD_b.js.map
 homepage: https://github.com/brettshollenberger/easy_ml
 licenses:
 - MIT

data/app/models/easy_ml/datasources/polars_datasource.rb

@@ -1,69 +0,0 @@
-module EasyML
-  module Datasources
-    class PolarsDatasource < BaseDatasource
-      include EasyML::DataframeSerialization
-
-      validates :df, presence: true
-      add_configuration_attributes :df
-
-      def query(**kwargs)
-        EasyML::Data::PolarsInMemory.query(df, **kwargs)
-      end
-
-      def in_batches(of: 10_000)
-        total_rows = df.shape[0]
-        (0...total_rows).step(of) do |start|
-          end_index = [start + of, total_rows].min
-          yield df.slice(start, end_index - start)
-        end
-      end
-
-      def all_files
-        []
-      end
-
-      def files
-        []
-      end
-
-      def last_updated_at
-        datasource.updated_at
-      end
-
-      def sha
-        nil
-      end
-
-      def data
-        df
-      end
-
-      def df
-        datasource.df
-      end
-
-      def exists?
-        df.present?
-      end
-
-      def error_not_exists
-        "Must have a dataframe"
-      end
-
-      def store_in_configuration
-        return unless df
-
-        datasource.configuration = (datasource.configuration || {}).merge(
-          "df" => serialize_dataframe(df),
-        )
-      end
-
-      def read_from_configuration
-        return unless datasource.configuration&.key?("df")
-
-        df_data = datasource.configuration["df"]
-        datasource.df = deserialize_dataframe(df_data)
-      end
-    end
-  end
-end

data/lib/easy_ml/data/filter_extensions.rb

@@ -1,31 +0,0 @@
-module EasyML
-  module Data
-    module FilterExtensions
-      def is_primary_key_filter?(primary_key)
-        return false unless primary_key
-        primary_key = [primary_key] unless primary_key.is_a?(Array)
-        # Filter expressions in Polars are represented as strings like:
-        # [([(col("LOAN_APP_ID")) > (dyn int: 4)]) & ([(col("LOAN_APP_ID")) < (dyn int: 16)])]
-        expr_str = to_s
-        return false unless expr_str.include?(primary_key.first)
-
-        # Check for common primary key operations
-        primary_key_ops = [">", "<", ">=", "<=", "=", "eq", "gt", "lt", "ge", "le"]
-        primary_key_ops.any? { |op| expr_str.include?(op) }
-      end
-
-      def extract_primary_key_values
-        expr_str = to_s
-        # Extract numeric values from the expression
-        # This will match both integers and floats
-        values = expr_str.scan(/(?:dyn int|float): (-?\d+(?:\.\d+)?)/).flatten.map(&:to_f)
-        values.uniq
-      end
-    end
-  end
-end
-
-# Extend Polars classes with our filter functionality
-[Polars::Expr].each do |klass|
-  klass.include(EasyML::Data::FilterExtensions)
-end