easy_ml 0.2.0.pre.rc63 → 0.2.0.pre.rc68

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 30ec865849f66547055d1835ccc18753308d9a6cd3c8a5862a01a20151b2e772
-  data.tar.gz: df9c0eee094ea607370bd84f97f2f269919e08e07fac1a4e0a17f96bc0286093
+  metadata.gz: 2404bc2b6613d95627dc89265e76351f648d9b45e65f5935a71a0e2334d08d2c
+  data.tar.gz: 072f1d58d6e5d7864e3cdb365701c6fb6e635bc20e3e9ef415fe52772671ce76
 SHA512:
-  metadata.gz: 6241d23ced8da76c9ca7e6789bfeccc75824b9e217fa01850f6ab6475ee77716b222d4c0661e965d69de8b95d9719d15ed67a2f1835c359ca3d186d193058b2c
-  data.tar.gz: a92fdc6934bf28d0f6a18f5356d8ac76f5ee0195f96fbed9110dc1862c976922ec7e1141b65e7050d80c1b63e5a90e7a5903147f9001167e7e12e7c553e9339c
+  metadata.gz: d4e8ab597db1470630e555448fb057af07fab38ff5125fd7ce548db5ba71ca7abac26d1761fb892459890b5c8bce169cd9e165d80b4b91ec6a806dda1a36cbe5
+  data.tar.gz: 71cab456d196c15e03b16e2677cb3b64e08527b2b4acc20ac8ee6c0f0ceea671b74df0d61c174a914cfa85f7e476ad77f264ae78e6b604ed8f50ca2756d84956

data/app/jobs/easy_ml/batch_job.rb

@@ -40,14 +40,23 @@ module EasyML
           Resque.redis.rpush("batch:#{parent_id}:remaining", batch.to_json)
         end
 
-        enqueue_batch(batch)
+        handle_batch(parent_id, batch)
+      end
+
+      def handle_batch(parent_id, batch)
+        if batch.size > 1
+          enqueue_batch(batch)
+        else
+          run_one_batch(parent_id, batch.first)
+          after_batch_hook(parent_id, batch)
+        end
       end
 
       def enqueue_next_batch(caller, parent_id)
         next_batch = Resque.redis.lpop("batch:#{parent_id}:remaining")
         payload = Resque.decode(next_batch)
 
-        caller.enqueue_batch(payload)
+        caller.handle_batch(parent_id, payload)
       end
 
       def next_batch?(parent_id)

data/app/jobs/easy_ml/compute_feature_job.rb

@@ -4,37 +4,21 @@ module EasyML
 
     @queue = :easy_ml
 
-    def self.perform(batch_id, options = {})
-      begin
-        options.symbolize_keys!
-        feature_id = options.dig(:feature_id)
-        feature = EasyML::Feature.find(feature_id)
-        dataset = feature.dataset
-
-        # Check if any feature has failed before proceeding
-        if dataset.features.any? { |f| f.workflow_status == "failed" }
-          return
-        end
-
-        feature.update(workflow_status: :analyzing) if feature.workflow_status == :ready
-        feature.fit_batch(options.merge!(batch_id: batch_id))
-      rescue => e
-        EasyML::Feature.transaction do
-          return if dataset.reload.workflow_status == :failed
-
-          feature.update(workflow_status: :failed)
-          dataset.update(workflow_status: :failed)
-          build_error_with_context(dataset, e, batch_id, feature)
-        end
-      end
+    def self.perform(batch_id, batch_args = {})
+      # This is very, very, very, very, very important
+      # if you don't dup the batch_args, resque-batched-job will
+      # fail in some non-obvious ways, because it will try to
+      # decode to match the original batch args EXACTLY.
+      #
+      # This will waste your time so please just don't remove this .dup!!!
+      #
+      # https://github.com/drfeelngood/resque-batched-job/blob/master/lib/resque/plugins/batched_job.rb#L86
+      batch_args = batch_args.dup
+      run_one_batch(batch_id, batch_args)
     end
 
-    def self.build_error_with_context(dataset, error, batch_id, feature)
-      error = EasyML::Event.handle_error(dataset, error)
-      batch = feature.build_batch(batch_id: batch_id)
-
-      # Convert any dataframes in the context to serialized form
-      error.create_context(context: batch)
+    def self.run_one_batch(batch_id, batch_args)
+      EasyML::Feature.fit_one_batch(batch_id, batch_args)
     end
 
     def self.after_batch_hook(batch_id, *args)

data/app/jobs/easy_ml/refresh_dataset_job.rb

@@ -16,14 +16,14 @@ module EasyML
         if dataset.features.needs_fit.any?
           dataset.fit_features(async: true)
         else
-          dataset.actually_refresh
+          dataset.after_fit_features
         end
       rescue StandardError => e
         if Rails.env.test?
           raise e
         end
         dataset.update(workflow_status: :failed)
-        handle_error(dataset, e)
+        EasyML::Event.handle_error(dataset, e)
       end
     end
   end

data/app/models/easy_ml/column/imputers/imputer.rb

@@ -8,7 +8,6 @@ module EasyML
           @column = column
           @dataset = column.dataset
           @preprocessing_step = preprocessing_step.with_indifferent_access
-          validate_preprocessing_step!
         end
 
         def inspect
@@ -72,31 +71,6 @@ module EasyML
 
           EasyML::Column::Imputers::OrdinalEncoder.new(column, preprocessing_step).decode_labels(df)
         end
-
-        private
-
-        def validate_preprocessing_step!
-          validate_params!
-          validate_method!
-        end
-
-        def validate_params!
-          return unless preprocessing_step[:params]
-
-          preprocessing_step[:params].keys.each do |param|
-            unless Imputers.supported_params.include?(param.to_sym)
-              raise ArgumentError, "Unsupported preprocessing parameter '#{param}'. Supported parameters are: #{Imputers.supported_params.join(", ")}"
-            end
-          end
-        end
-
-        def validate_method!
-          return unless preprocessing_step[:method]
-
-          unless Imputers.supported_methods.include?(preprocessing_step[:method].to_sym)
-            raise ArgumentError, "Unsupported preprocessing method '#{preprocessing_step[:method]}'. Supported methods are: #{Imputers.supported_methods.join(", ")}"
-          end
-        end
       end
     end
   end

data/app/models/easy_ml/column/selector.rb

@@ -45,7 +45,7 @@ module EasyML
       end
 
       def data(**kwargs)
-        if column.is_computed?
+        if column.is_computed? && !column.in_raw_dataset?
           Selector.new(column, :processed).send(:select, :data, **kwargs)
         else
           select(:data, **kwargs)

data/app/models/easy_ml/column.rb

@@ -43,6 +43,7 @@ module EasyML
     before_save :set_defaults
     before_save :set_feature_lineage
     before_save :set_polars_datatype
+    after_find :ensure_feature_exists
 
     # Scopes
     scope :visible, -> { where(hidden: false) }
@@ -108,6 +109,13 @@ module EasyML
           }
     scope :is_learning, -> { where(is_learning: true) }
 
+    def ensure_feature_exists
+      if feature && !feature.has_code?
+        feature.destroy
+        update(feature_id: nil)
+      end
+    end
+
     def display_attributes
       attributes.except(:statistics)
     end
@@ -206,9 +214,10 @@ module EasyML
 
     def raw_dtype
       return @raw_dtype if @raw_dtype
+      set_feature_lineage
 
       if in_raw_dataset?
-        @raw_dtype = raw&.data&.to_series&.dtype
+        @raw_dtype = raw&.data&.to_series.try(:dtype)
       elsif already_computed?
         @raw_dtype = processed&.data&.to_series&.dtype
       end
@@ -284,6 +293,8 @@ module EasyML
     alias_method :feature, :computing_feature
 
     def set_feature_lineage
+      return if dataset.nil?
+
       if dataset.features.computed_column_names.include?(name)
         if computed_by.nil?
           assign_attributes(

data/app/models/easy_ml/dataset.rb

@@ -202,7 +202,6 @@ module EasyML
         prepare!
         fit_features!(async: async)
       end
-      after_fit_features unless async
     end
 
     def refresh(async: false)
@@ -212,7 +211,6 @@ module EasyML
         prepare
         fit_features(async: async)
       end
-      after_fit_features unless async
     end
 
     def fit_features!(async: false, features: self.features)
@@ -221,7 +219,7 @@ module EasyML
 
     def fit_features(async: false, features: self.features, force: false)
       features_to_compute = force ? features : features.needs_fit
-      return if features_to_compute.empty?
+      return after_fit_features if features_to_compute.empty?
 
       features.first.fit(features: features_to_compute, async: async)
     end
@@ -231,7 +229,6 @@ module EasyML
       reload
       return if failed?
 
-      features.update_all(needs_fit: false, fit_at: Time.current)
       actually_refresh
     end
 

data/app/models/easy_ml/feature.rb

@@ -82,7 +82,7 @@ module EasyML
             where(id: fittable.map(&:id))
           end
     scope :needs_fit, -> { has_changes.or(never_applied).or(never_fit) }
-    scope :ready_to_apply, -> { where.not(id: needs_fit.map(&:id)) }
+    scope :ready_to_apply, -> { where(needs_fit: false).where.not(id: has_changes.map(&:id)) }
 
     before_save :apply_defaults, if: :new_record?
     before_save :update_sha
@@ -95,6 +95,10 @@ module EasyML
       raise InvalidFeatureError, "Invalid feature class: #{feature_class}"
     end
 
+    def has_code?
+      feature_klass.present?
+    end
+
     def adapter
       @adapter ||= feature_klass.new
     end
@@ -232,13 +236,54 @@ module EasyML
       if async && job_count > 1
         EasyML::ComputeFeatureJob.enqueue_ordered_batches(jobs)
       else
-        jobs.flatten.each do |job|
-          EasyML::ComputeFeatureJob.perform(nil, job)
+        jobs.each do |feature_batch|
+          feature_batch.each do |batch_args|
+            EasyML::ComputeFeatureJob.perform(nil, batch_args)
+          end
+          feature = EasyML::Feature.find(feature_batch.first.dig(:feature_id))
+          feature.after_fit
         end
-        features.each(&:after_fit) unless features.any?(&:failed?)
+        dataset.after_fit_features
       end
     end
 
+    def self.fit_one_batch(batch_id, batch_args = {})
+      batch_args.symbolize_keys!
+      feature_id = batch_args.dig(:feature_id)
+      feature = EasyML::Feature.find(feature_id)
+      dataset = feature.dataset
+
+      # Check if any feature has failed before proceeding
+      return if dataset.features.any? { |f| f.workflow_status == "failed" }
+
+      feature.update(workflow_status: :analyzing) if feature.workflow_status == :ready
+      begin
+        feature.fit_batch(batch_args.merge!(batch_id: batch_id))
+      rescue => e
+        EasyML::Feature.transaction do
+          return if dataset.reload.workflow_status == :failed
+
+          feature.update(workflow_status: :failed)
+          dataset.update(workflow_status: :failed)
+          build_error_with_context(dataset, e, batch_id, feature)
+        end
+        raise e
+      end
+    end
+
+    def self.build_error_with_context(dataset, error, batch_id, feature)
+      error = EasyML::Event.handle_error(dataset, error)
+      batch = feature.build_batch(batch_id: batch_id)
+
+      # Convert any dataframes in the context to serialized form
+      error.create_context(context: batch)
+    end
+
+    def self.fit_feature_failed(dataset, e)
+      dataset.update(workflow_status: :failed)
+      EasyML::Event.handle_error(dataset, e)
+    end
+
     # Fit a single batch, used for testing the user's feature implementation
     def fit_batch(batch_args = {})
       batch_args.symbolize_keys!
@@ -301,7 +346,11 @@ module EasyML
       end
       return if df.blank?
 
-      batch_df = adapter.fit(df, self, batch_args)
+      begin
+        batch_df = adapter.fit(df, self, batch_args)
+      rescue => e
+        raise "Feature #{feature_class}#fit failed: #{e.message}"
+      end
       if batch_df.present?
         store(batch_df)
       else
@@ -315,7 +364,11 @@ module EasyML
       return df if !adapter.respond_to?(:transform) && feature_store.empty?
 
       df_len_was = df.shape[0]
-      result = adapter.transform(df, self)
+      begin
+        result = adapter.transform(df, self)
+      rescue => e
+        raise "Feature #{feature_class}#transform failed: #{e.message}"
+      end
       raise "Feature '#{name}' must return a Polars::DataFrame, got #{result.class}" unless result.is_a?(Polars::DataFrame)
       df_len_now = result.shape[0]
       raise "Feature #{feature_class}#transform: output size must match input size! Input size: #{df_len_now}, output size: #{df_len_was}." if df_len_now != df_len_was
@@ -411,8 +464,10 @@ module EasyML
     end
 
     def after_fit
+      update_sha
+
       updates = {
-        applied_at: Time.current,
+        fit_at: Time.current,
         needs_fit: false,
         workflow_status: :ready,
       }.compact
@@ -472,7 +527,11 @@ module EasyML
     end
 
     def feature_klass
-      @feature_klass ||= EasyML::Features::Registry.find(feature_class.to_s).dig(:feature_class).constantize
+      begin
+        @feature_klass ||= EasyML::Features::Registry.find(feature_class.to_s).dig(:feature_class).constantize
+      rescue => e
+        nil
+      end
     end
 
     def config

data/lib/easy_ml/feature_store.rb

@@ -15,10 +15,10 @@ module EasyML
       max_key = df[primary_key].max
       batch_size = feature.batch_size || 10_000
 
-      # Try to parse as integers if they're strings
       begin
-        min_key = min_key.to_i if min_key.is_a?(String)
-        max_key = max_key.to_i if max_key.is_a?(String)
+        # We are intentionally not using to_i, so it will raise an error for keys like "A1"
+        min_key = Integer(min_key) if min_key.is_a?(String)
+        max_key = Integer(max_key) if max_key.is_a?(String)
       rescue ArgumentError
         return store_without_partitioning(df)
       end

data/lib/easy_ml/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module EasyML
-  VERSION = "0.2.0-rc63"
+  VERSION = "0.2.0-rc68"
 
   module Version
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: easy_ml
 version: !ruby/object:Gem::Version
-  version: 0.2.0.pre.rc63
+  version: 0.2.0.pre.rc68
 platform: ruby
 authors:
 - Brett Shollenberger
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-02-06 00:00:00.000000000 Z
+date: 2025-02-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -553,7 +553,6 @@ files:
 - app/jobs/easy_ml/clean_job.rb
 - app/jobs/easy_ml/compute_feature_job.rb
 - app/jobs/easy_ml/deploy_job.rb
-- app/jobs/easy_ml/finalize_feature_job.rb
 - app/jobs/easy_ml/refresh_dataset_job.rb
 - app/jobs/easy_ml/schedule_retraining_job.rb
 - app/jobs/easy_ml/sync_datasource_job.rb

data/app/jobs/easy_ml/finalize_feature_job.rb

@@ -1,13 +0,0 @@
-module EasyML
-  class FinalizeFeatureJob < ApplicationJob
-    queue_as :features
-
-    def perform(feature_id)
-      feature = EasyML::Feature.find(feature_id)
-      feature.update!(
-        applied_at: Time.current,
-        needs_fit: false,
-      )
-    end
-  end
-end