easy_ml 0.2.0.pre.rc65 → 0.2.0.pre.rc68

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d6993d639004ee88981816cf11422f458d2fa5caa121e760d075c7a73ae70195
-  data.tar.gz: 0e60804c7d59f8c3402be88b6b6ae5e24a7c9542875cbc29f606bbd500227b1f
+  metadata.gz: 2404bc2b6613d95627dc89265e76351f648d9b45e65f5935a71a0e2334d08d2c
+  data.tar.gz: 072f1d58d6e5d7864e3cdb365701c6fb6e635bc20e3e9ef415fe52772671ce76
 SHA512:
-  metadata.gz: b11150da87d6dafb5d0a71f0c9b8391012a388ba51eb17544dca044d8616b2d9898fa65ef37e7c8cb22f627669e648b23edc37f6582f5aa40d4619228c57ed02
-  data.tar.gz: 7fd29a43e9a2a15b3388e2c592fe5772d15e394103d7c0651fbe0404abd7bd8637e8ff32f3c38cac4220ce815ab49f369c5767d00dab0cd5ed060a924d3fa8bb
+  metadata.gz: d4e8ab597db1470630e555448fb057af07fab38ff5125fd7ce548db5ba71ca7abac26d1761fb892459890b5c8bce169cd9e165d80b4b91ec6a806dda1a36cbe5
+  data.tar.gz: 71cab456d196c15e03b16e2677cb3b64e08527b2b4acc20ac8ee6c0f0ceea671b74df0d61c174a914cfa85f7e476ad77f264ae78e6b604ed8f50ca2756d84956

data/app/jobs/easy_ml/compute_feature_job.rb

@@ -5,40 +5,22 @@ module EasyML
     @queue = :easy_ml
 
     def self.perform(batch_id, batch_args = {})
-      begin
-        # This is very, very, very, very, very important
-        # if you don't dup the batch_args, resque-batched-job will
-        # fail in some non-obvious ways, because it will try to
-        # decode to match the original batch args EXACTLY.
-        #
-        # This will waste your time so please just don't remove this .dup!!!
-        #
-        # https://github.com/drfeelngood/resque-batched-job/blob/master/lib/resque/plugins/batched_job.rb#L86
-        batch_args = batch_args.dup
-        run_one_batch(batch_id, batch_args)
-      rescue => e
-        EasyML::Feature.transaction do
-          return if dataset.reload.workflow_status == :failed
-
-          feature.update(workflow_status: :failed)
-          dataset.update(workflow_status: :failed)
-          build_error_with_context(dataset, e, batch_id, feature)
-        end
-      end
+      # This is very, very, very, very, very important
+      # if you don't dup the batch_args, resque-batched-job will
+      # fail in some non-obvious ways, because it will try to
+      # decode to match the original batch args EXACTLY.
+      #
+      # This will waste your time so please just don't remove this .dup!!!
+      #
+      # https://github.com/drfeelngood/resque-batched-job/blob/master/lib/resque/plugins/batched_job.rb#L86
+      batch_args = batch_args.dup
+      run_one_batch(batch_id, batch_args)
     end
 
     def self.run_one_batch(batch_id, batch_args)
       EasyML::Feature.fit_one_batch(batch_id, batch_args)
     end
 
-    def self.build_error_with_context(dataset, error, batch_id, feature)
-      error = EasyML::Event.handle_error(dataset, error)
-      batch = feature.build_batch(batch_id: batch_id)
-
-      # Convert any dataframes in the context to serialized form
-      error.create_context(context: batch)
-    end
-
     def self.after_batch_hook(batch_id, *args)
       batch_args = fetch_batch_arguments(batch_id).flatten.map(&:symbolize_keys)
       feature_ids = batch_args.pluck(:feature_id).uniq

data/app/models/easy_ml/column.rb

@@ -43,6 +43,7 @@ module EasyML
     before_save :set_defaults
     before_save :set_feature_lineage
     before_save :set_polars_datatype
+    after_find :ensure_feature_exists
 
     # Scopes
     scope :visible, -> { where(hidden: false) }
@@ -108,6 +109,13 @@ module EasyML
           }
     scope :is_learning, -> { where(is_learning: true) }
 
+    def ensure_feature_exists
+      if feature && !feature.has_code?
+        feature.destroy
+        update(feature_id: nil)
+      end
+    end
+
     def display_attributes
       attributes.except(:statistics)
     end
@@ -285,6 +293,8 @@ module EasyML
     alias_method :feature, :computing_feature
 
     def set_feature_lineage
+      return if dataset.nil?
+
       if dataset.features.computed_column_names.include?(name)
         if computed_by.nil?
           assign_attributes(

data/app/models/easy_ml/dataset.rb

@@ -219,13 +219,12 @@ module EasyML
 
     def fit_features(async: false, features: self.features, force: false)
       features_to_compute = force ? features : features.needs_fit
-      return if features_to_compute.empty?
+      return after_fit_features if features_to_compute.empty?
 
       features.first.fit(features: features_to_compute, async: async)
     end
 
     def after_fit_features
-      puts "AFTER FIT FEATURES!"
       unlock!
       reload
       return if failed?

data/app/models/easy_ml/feature.rb

@@ -82,7 +82,7 @@ module EasyML
             where(id: fittable.map(&:id))
           end
     scope :needs_fit, -> { has_changes.or(never_applied).or(never_fit) }
-    scope :ready_to_apply, -> { where.not(id: needs_fit.map(&:id)) }
+    scope :ready_to_apply, -> { where(needs_fit: false).where.not(id: has_changes.map(&:id)) }
 
     before_save :apply_defaults, if: :new_record?
     before_save :update_sha
@@ -95,6 +95,10 @@ module EasyML
       raise InvalidFeatureError, "Invalid feature class: #{feature_class}"
     end
 
+    def has_code?
+      feature_klass.present?
+    end
+
     def adapter
       @adapter ||= feature_klass.new
     end
@@ -250,18 +254,31 @@ module EasyML
       dataset = feature.dataset
 
       # Check if any feature has failed before proceeding
-      if dataset.features.any? { |f| f.workflow_status == "failed" }
-        return
-      end
+      return if dataset.features.any? { |f| f.workflow_status == "failed" }
+
       feature.update(workflow_status: :analyzing) if feature.workflow_status == :ready
       begin
         feature.fit_batch(batch_args.merge!(batch_id: batch_id))
       rescue => e
-        EasyML::Feature.fit_feature_failed(dataset, e)
+        EasyML::Feature.transaction do
+          return if dataset.reload.workflow_status == :failed
+
+          feature.update(workflow_status: :failed)
+          dataset.update(workflow_status: :failed)
+          build_error_with_context(dataset, e, batch_id, feature)
+        end
         raise e
       end
     end
 
+    def self.build_error_with_context(dataset, error, batch_id, feature)
+      error = EasyML::Event.handle_error(dataset, error)
+      batch = feature.build_batch(batch_id: batch_id)
+
+      # Convert any dataframes in the context to serialized form
+      error.create_context(context: batch)
+    end
+
     def self.fit_feature_failed(dataset, e)
       dataset.update(workflow_status: :failed)
       EasyML::Event.handle_error(dataset, e)
@@ -447,6 +464,8 @@ module EasyML
     end
 
     def after_fit
+      update_sha
+
       updates = {
         fit_at: Time.current,
         needs_fit: false,
@@ -508,7 +527,11 @@ module EasyML
     end
 
     def feature_klass
-      @feature_klass ||= EasyML::Features::Registry.find(feature_class.to_s).dig(:feature_class).constantize
+      begin
+        @feature_klass ||= EasyML::Features::Registry.find(feature_class.to_s).dig(:feature_class).constantize
+      rescue => e
+        nil
+      end
     end
 
     def config

data/lib/easy_ml/feature_store.rb

@@ -15,10 +15,10 @@ module EasyML
       max_key = df[primary_key].max
       batch_size = feature.batch_size || 10_000
 
-      # Try to parse as integers if they're strings
       begin
-        min_key = min_key.to_i if min_key.is_a?(String)
-        max_key = max_key.to_i if max_key.is_a?(String)
+        # We are intentionally not using to_i, so it will raise an error for keys like "A1"
+        min_key = Integer(min_key) if min_key.is_a?(String)
+        max_key = Integer(max_key) if max_key.is_a?(String)
       rescue ArgumentError
         return store_without_partitioning(df)
       end

data/lib/easy_ml/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module EasyML
-  VERSION = "0.2.0-rc65"
+  VERSION = "0.2.0-rc68"
 
   module Version
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: easy_ml
 version: !ruby/object:Gem::Version
-  version: 0.2.0.pre.rc65
+  version: 0.2.0.pre.rc68
 platform: ruby
 authors:
 - Brett Shollenberger
@@ -553,7 +553,6 @@ files:
 - app/jobs/easy_ml/clean_job.rb
 - app/jobs/easy_ml/compute_feature_job.rb
 - app/jobs/easy_ml/deploy_job.rb
-- app/jobs/easy_ml/finalize_feature_job.rb
 - app/jobs/easy_ml/refresh_dataset_job.rb
 - app/jobs/easy_ml/schedule_retraining_job.rb
 - app/jobs/easy_ml/sync_datasource_job.rb

data/app/jobs/easy_ml/finalize_feature_job.rb

@@ -1,13 +0,0 @@
-module EasyML
-  class FinalizeFeatureJob < ApplicationJob
-    queue_as :features
-
-    def perform(feature_id)
-      feature = EasyML::Feature.find(feature_id)
-      feature.update!(
-        applied_at: Time.current,
-        needs_fit: false,
-      )
-    end
-  end
-end