easy_ml 0.2.0.pre.rc62 → 0.2.0.pre.rc65

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ecfdf6e2547bfaeab500da7ce4cc8ba417dbab1eac64ed74c422fa1c3bf3db5c
-  data.tar.gz: be43f29853adc807cf9acb6ca92aee54d237d17fb3e858ca39683227ab976241
+  metadata.gz: d6993d639004ee88981816cf11422f458d2fa5caa121e760d075c7a73ae70195
+  data.tar.gz: 0e60804c7d59f8c3402be88b6b6ae5e24a7c9542875cbc29f606bbd500227b1f
 SHA512:
-  metadata.gz: 7f1c9ae555b2edc3957836395630c1515ad47f2429f1037e38b06c8a14e48a46fdc460fa7816c9686c7590a21ddfe92ff93f84068a8e5dff8fc36adc81fdbd14
-  data.tar.gz: e1f132a13141f76f4af4ae22893f4f6d2451b91bbaa3fcc344c3f49eb35bac21f13325460b002f84913e903b359314949817307c90cfe9ba6c0a622e5e0ea163
+  metadata.gz: b11150da87d6dafb5d0a71f0c9b8391012a388ba51eb17544dca044d8616b2d9898fa65ef37e7c8cb22f627669e648b23edc37f6582f5aa40d4619228c57ed02
+  data.tar.gz: 7fd29a43e9a2a15b3388e2c592fe5772d15e394103d7c0651fbe0404abd7bd8637e8ff32f3c38cac4220ce815ab49f369c5767d00dab0cd5ed060a924d3fa8bb

data/app/jobs/easy_ml/batch_job.rb

@@ -40,14 +40,23 @@ module EasyML
           Resque.redis.rpush("batch:#{parent_id}:remaining", batch.to_json)
         end
 
-        enqueue_batch(batch)
+        handle_batch(parent_id, batch)
+      end
+
+      def handle_batch(parent_id, batch)
+        if batch.size > 1
+          enqueue_batch(batch)
+        else
+          run_one_batch(parent_id, batch.first)
+          after_batch_hook(parent_id, batch)
+        end
       end
 
       def enqueue_next_batch(caller, parent_id)
         next_batch = Resque.redis.lpop("batch:#{parent_id}:remaining")
         payload = Resque.decode(next_batch)
 
-        caller.enqueue_batch(payload)
+        caller.handle_batch(parent_id, payload)
       end
 
       def next_batch?(parent_id)

data/app/jobs/easy_ml/compute_feature_job.rb

@@ -4,20 +4,18 @@ module EasyML
 
     @queue = :easy_ml
 
-    def self.perform(batch_id, options = {})
+    def self.perform(batch_id, batch_args = {})
       begin
-        options.symbolize_keys!
-        feature_id = options.dig(:feature_id)
-        feature = EasyML::Feature.find(feature_id)
-        dataset = feature.dataset
-
-        # Check if any feature has failed before proceeding
-        if dataset.features.any? { |f| f.workflow_status == "failed" }
-          return
-        end
-
-        feature.update(workflow_status: :analyzing) if feature.workflow_status == :ready
-        feature.fit_batch(options.merge!(batch_id: batch_id))
+        # This is very, very, very, very, very important
+        # if you don't dup the batch_args, resque-batched-job will
+        # fail in some non-obvious ways, because it will try to
+        # decode to match the original batch args EXACTLY.
+        #
+        # This will waste your time so please just don't remove this .dup!!!
+        #
+        # https://github.com/drfeelngood/resque-batched-job/blob/master/lib/resque/plugins/batched_job.rb#L86
+        batch_args = batch_args.dup
+        run_one_batch(batch_id, batch_args)
       rescue => e
         EasyML::Feature.transaction do
           return if dataset.reload.workflow_status == :failed
@@ -29,6 +27,10 @@ module EasyML
       end
     end
 
+    def self.run_one_batch(batch_id, batch_args)
+      EasyML::Feature.fit_one_batch(batch_id, batch_args)
+    end
+
     def self.build_error_with_context(dataset, error, batch_id, feature)
       error = EasyML::Event.handle_error(dataset, error)
       batch = feature.build_batch(batch_id: batch_id)

data/app/jobs/easy_ml/refresh_dataset_job.rb

@@ -16,14 +16,14 @@ module EasyML
         if dataset.features.needs_fit.any?
           dataset.fit_features(async: true)
         else
-          dataset.actually_refresh
+          dataset.after_fit_features
         end
       rescue StandardError => e
         if Rails.env.test?
           raise e
         end
         dataset.update(workflow_status: :failed)
-        handle_error(dataset, e)
+        EasyML::Event.handle_error(dataset, e)
       end
     end
   end

data/app/models/easy_ml/column/imputers/imputer.rb

@@ -8,7 +8,6 @@ module EasyML
           @column = column
           @dataset = column.dataset
           @preprocessing_step = preprocessing_step.with_indifferent_access
-          validate_preprocessing_step!
         end
 
         def inspect
@@ -72,31 +71,6 @@ module EasyML
 
           EasyML::Column::Imputers::OrdinalEncoder.new(column, preprocessing_step).decode_labels(df)
         end
-
-        private
-
-        def validate_preprocessing_step!
-          validate_params!
-          validate_method!
-        end
-
-        def validate_params!
-          return unless preprocessing_step[:params]
-
-          preprocessing_step[:params].keys.each do |param|
-            unless Imputers.supported_params.include?(param.to_sym)
-              raise ArgumentError, "Unsupported preprocessing parameter '#{param}'. Supported parameters are: #{Imputers.supported_params.join(", ")}"
-            end
-          end
-        end
-
-        def validate_method!
-          return unless preprocessing_step[:method]
-
-          unless Imputers.supported_methods.include?(preprocessing_step[:method].to_sym)
-            raise ArgumentError, "Unsupported preprocessing method '#{preprocessing_step[:method]}'. Supported methods are: #{Imputers.supported_methods.join(", ")}"
-          end
-        end
       end
     end
   end

data/app/models/easy_ml/column/lineage/computed_by_feature.rb

@@ -11,7 +11,7 @@ module EasyML
         end
 
         def timestamp
-          column.feature.fit_at
+          column.feature.fit_at || column.feature.applied_at
         end
 
         def check

data/app/models/easy_ml/column/selector.rb

@@ -45,7 +45,7 @@ module EasyML
       end
 
       def data(**kwargs)
-        if column.is_computed?
+        if column.is_computed? && !column.in_raw_dataset?
           Selector.new(column, :processed).send(:select, :data, **kwargs)
         else
           select(:data, **kwargs)

data/app/models/easy_ml/column.rb

@@ -141,7 +141,13 @@ module EasyML
         assign_attributes(datatype: processed.data.to_series.dtype)
       end
       set_sample_values
-      assign_attributes(statistics: (read_attribute(:statistics) || {}).symbolize_keys.merge!(learner.learn(type: type).symbolize_keys))
+      new_stats = learner.learn(type: type).symbolize_keys
+
+      if !in_raw_dataset?
+        new_stats[:raw] = new_stats[:processed]
+      end
+
+      assign_attributes(statistics: (read_attribute(:statistics) || {}).symbolize_keys.merge!(new_stats))
       assign_attributes(
         learned_at: UTC.now,
         last_datasource_sha: dataset.last_datasource_sha,
@@ -200,9 +206,10 @@ module EasyML
 
     def raw_dtype
       return @raw_dtype if @raw_dtype
+      set_feature_lineage
 
       if in_raw_dataset?
-        @raw_dtype = raw&.data&.to_series&.dtype
+        @raw_dtype = raw&.data&.to_series.try(:dtype)
       elsif already_computed?
         @raw_dtype = processed&.data&.to_series&.dtype
       end
@@ -410,9 +417,9 @@ module EasyML
 
       case datatype&.to_sym
       when :float
-        Float(value)
+        value.to_f
       when :integer
-        Integer(value)
+        value.to_i
       when :boolean
         ActiveModel::Type::Boolean.new.cast(value)
       when :datetime

data/app/models/easy_ml/dataset.rb

@@ -202,7 +202,6 @@ module EasyML
         prepare!
         fit_features!(async: async)
       end
-      after_fit_features unless async
     end
 
     def refresh(async: false)
@@ -212,7 +211,6 @@ module EasyML
         prepare
         fit_features(async: async)
       end
-      after_fit_features unless async
     end
 
     def fit_features!(async: false, features: self.features)
@@ -227,11 +225,11 @@ module EasyML
     end
 
     def after_fit_features
+      puts "AFTER FIT FEATURES!"
       unlock!
       reload
       return if failed?
 
-      features.update_all(needs_fit: false, fit_at: Time.current)
       actually_refresh
     end
 

data/app/models/easy_ml/feature.rb

@@ -232,13 +232,41 @@ module EasyML
       if async && job_count > 1
         EasyML::ComputeFeatureJob.enqueue_ordered_batches(jobs)
       else
-        jobs.flatten.each do |job|
-          EasyML::ComputeFeatureJob.perform(nil, job)
+        jobs.each do |feature_batch|
+          feature_batch.each do |batch_args|
+            EasyML::ComputeFeatureJob.perform(nil, batch_args)
+          end
+          feature = EasyML::Feature.find(feature_batch.first.dig(:feature_id))
+          feature.after_fit
         end
-        features.each(&:after_fit) unless features.any?(&:failed?)
+        dataset.after_fit_features
       end
     end
 
+    def self.fit_one_batch(batch_id, batch_args = {})
+      batch_args.symbolize_keys!
+      feature_id = batch_args.dig(:feature_id)
+      feature = EasyML::Feature.find(feature_id)
+      dataset = feature.dataset
+
+      # Check if any feature has failed before proceeding
+      if dataset.features.any? { |f| f.workflow_status == "failed" }
+        return
+      end
+      feature.update(workflow_status: :analyzing) if feature.workflow_status == :ready
+      begin
+        feature.fit_batch(batch_args.merge!(batch_id: batch_id))
+      rescue => e
+        EasyML::Feature.fit_feature_failed(dataset, e)
+        raise e
+      end
+    end
+
+    def self.fit_feature_failed(dataset, e)
+      dataset.update(workflow_status: :failed)
+      EasyML::Event.handle_error(dataset, e)
+    end
+
     # Fit a single batch, used for testing the user's feature implementation
     def fit_batch(batch_args = {})
       batch_args.symbolize_keys!
@@ -301,7 +329,11 @@ module EasyML
       end
       return if df.blank?
 
-      batch_df = adapter.fit(df, self, batch_args)
+      begin
+        batch_df = adapter.fit(df, self, batch_args)
+      rescue => e
+        raise "Feature #{feature_class}#fit failed: #{e.message}"
+      end
       if batch_df.present?
         store(batch_df)
       else
@@ -315,7 +347,11 @@ module EasyML
       return df if !adapter.respond_to?(:transform) && feature_store.empty?
 
       df_len_was = df.shape[0]
-      result = adapter.transform(df, self)
+      begin
+        result = adapter.transform(df, self)
+      rescue => e
+        raise "Feature #{feature_class}#transform failed: #{e.message}"
+      end
       raise "Feature '#{name}' must return a Polars::DataFrame, got #{result.class}" unless result.is_a?(Polars::DataFrame)
       df_len_now = result.shape[0]
       raise "Feature #{feature_class}#transform: output size must match input size! Input size: #{df_len_now}, output size: #{df_len_was}." if df_len_now != df_len_was
@@ -412,7 +448,7 @@ module EasyML
 
     def after_fit
       updates = {
-        applied_at: Time.current,
+        fit_at: Time.current,
         needs_fit: false,
         workflow_status: :ready,
       }.compact

data/lib/easy_ml/feature_store.rb

@@ -17,8 +17,8 @@ module EasyML
 
       # Try to parse as integers if they're strings
       begin
-        min_key = Integer(min_key) if min_key.is_a?(String)
-        max_key = Integer(max_key) if max_key.is_a?(String)
+        min_key = min_key.to_i if min_key.is_a?(String)
+        max_key = max_key.to_i if max_key.is_a?(String)
       rescue ArgumentError
         return store_without_partitioning(df)
       end

data/lib/easy_ml/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module EasyML
-  VERSION = "0.2.0-rc62"
+  VERSION = "0.2.0-rc65"
 
   module Version
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: easy_ml
 version: !ruby/object:Gem::Version
-  version: 0.2.0.pre.rc62
+  version: 0.2.0.pre.rc65
 platform: ruby
 authors:
 - Brett Shollenberger
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-02-06 00:00:00.000000000 Z
+date: 2025-02-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord