easy_ml 0.2.0.pre.rc91 → 0.2.0.pre.rc93

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3a12058c269a91c130f9158e1507c58dc94ad33517aabe568a2f0bc9f78b88eb
-  data.tar.gz: 6a37a568b6a8d8c100c21dea96487cb85494ef11ba3b30ac51aad8cab45654d7
+  metadata.gz: f402f4f0d35dd702c8e97c841f22681626cff3c1ac030991cde884788b31833b
+  data.tar.gz: 1666649bca8f2ab7bae0ec7a5faa3a4561f85db5b22ffdda92e472f7661fab85
 SHA512:
-  metadata.gz: 4f344c00a9e2b557079943f7f6f2c4d7923dbb5b425423b81d58dbc6d63ac15d5e978d6f7e1ab2c02233fe8cc33b55d168ed1ffb0d1f2e1cfcc59670b812285d
-  data.tar.gz: 4831eac6b35b452b300408b37695d5116b6840404cf045e85d98780d3a120ae2d267b07852bc80b6918379b70f9c04a52a712e599751e469af72a0be6e1889c4
+  metadata.gz: fa77810edaf759b75240f57a8f6dcc99a54043326ba92ff95c6df6732bd614bb932a5558722d690544367b21c929755b249ff8ede56581bbb6c7abe37688db63
+  data.tar.gz: 7df62d34d6fd6dcd83eef86ab27dea0581bcbbe5c384c9fae00399661a62a882d3f163807171128d0efcc1026cd887789f961d9ce9e99ecaace8f91432f9193b

data/app/jobs/easy_ml/refresh_dataset_job.rb

@@ -14,18 +14,14 @@ module EasyML
         create_event(dataset, "started")
 
         dataset.unlock!
-        dataset.prepare
-        if dataset.features.needs_fit.any?
-          dataset.fit_features(async: true)
-        else
-          dataset.after_fit_features
+        dataset.refreshing do
+          dataset.prepare
+          if dataset.features.needs_fit.any?
+            dataset.fit_features(async: true)
+          else
+            dataset.after_fit_features
+          end
         end
-      rescue StandardError => e
-        if Rails.env.test?
-          raise e
-        end
-        dataset.update(workflow_status: :failed)
-        EasyML::Event.handle_error(dataset, e)
       end
     end
   end

data/app/models/easy_ml/column.rb

@@ -515,17 +515,17 @@ module EasyML
     end
 
     def cast_statement(df, df_col, expected_dtype)
-      expected_dtype = expected_dtype.is_a?(Polars::DataType) ? expected_dtype : expected_dtype.class
+      expected_dtype = expected_dtype.is_a?(Polars::DataType) ? expected_dtype.class : expected_dtype
       actual_type = df[df_col].dtype
 
-      cast_statement = case expected_dtype
-                       when Polars::Boolean
-                          case actual_type
-                          when Polars::Boolean
+      cast_statement = case expected_dtype.to_s
+                       when "Polars::Boolean"
+                          case actual_type.to_s
+                          when "Polars::Boolean"
                             Polars.col(df_col).cast(expected_dtype)
-                          when Polars::String, Polars::Categorical
+                          when "Polars::Utf8", "Polars::Categorical", "Polars::String"
                             Polars.col(df_col).eq("true").cast(expected_dtype)
-                          when Polars::Null
+                          when "Polars::Null"
                             Polars.col(df_col)
                           else
                             raise "Unexpected dtype: #{actual_type} for column: #{df_col}"

data/app/models/easy_ml/column_list.rb

@@ -104,11 +104,7 @@ module EasyML
         expected_dtype = schema[df_col.to_sym]
         db_col.cast_statement(df, df_col, expected_dtype)
       end
-      begin
-        df = df.with_columns(cast_statements)
-      rescue => e
-        binding.pry
-      end
+      df = df.with_columns(cast_statements)
     end
 
     def cast(processed_or_raw)
@@ -165,7 +161,8 @@ module EasyML
       end
       EasyML::Column.import(columns, on_duplicate_key_update: { columns: %i[ is_computed computed_by ] })
 
-      lineage = cols_to_learn.flat_map do |col|
+      cols = where(id: cols_to_learn.map(&:id)).includes(:lineages, :feature)
+      lineage = cols.flat_map do |col|
         EasyML::Lineage.learn(col)
       end.compact
       EasyML::Lineage.import(lineage, on_duplicate_key_update: { columns: %i[ column_id key occurred_at description ] })

data/app/models/easy_ml/dataset/learner.rb

@@ -2,18 +2,15 @@ module EasyML
   class Dataset
     class Learner
       include EasyML::Timing
-      attr_accessor :dataset, :columns, :type, :computed, :raw_columns, :statistics
+      attr_accessor :dataset, :columns, :all_columns, :type, :computed, :raw_columns, :statistics
 
       def initialize(dataset, type: :raw)
         @dataset = dataset
-        @columns = dataset.columns.reload.needs_learn.sort_by(&:name)
-
-        if computed
-          @columns = @columns.computed
-        end
-
-        @columns = @columns.select(&:persisted?).reject(&:empty?)
+        @columns = dataset.columns.reload.needs_learn.includes(:feature).sort_by(&:name)
         @type = type
+        @all_columns = @columns.dup
+        @columns = @columns.select(&:persisted?)
+        @columns = @columns.select { |c| available_columns.include?(c.name) }
       end
 
       def learn
@@ -23,6 +20,10 @@ module EasyML
         save_statistics
       end
 
+      def available_columns
+        @available_columns ||= dataset.send(type).data(lazy: true).schema.keys & columns.map(&:name)
+      end
+
       private
 
       def fit_models
@@ -35,10 +36,20 @@ module EasyML
         end
       end
 
+      def get_sample_values
+        needs_sample = EasyML::Column.where(id: columns.map(&:id)).where(sample_values: nil).map(&:name)
+        sampleable_cols = available_columns & needs_sample
+        selects = sampleable_cols.map do |col|
+          Polars.col(col).filter(Polars.col(col).is_not_null).limit(5).alias(col)
+        end
+        df = dataset.send(type).train(all_columns: true, lazy: true).select(selects).collect.to_h.transform_values(&:to_a)
+      end
+
       def save_statistics
-        columns.each do |col|
+        samples = get_sample_values
+        all_columns.each do |col|
           col.merge_statistics(statistics.dig(col.name))
-          col.set_sample_values
+          col.assign_attributes(sample_values: samples[col.name]) if samples[col.name].present?
           col.assign_attributes(
             learned_at: EasyML::Support::UTC.now,
             last_datasource_sha: col.dataset.last_datasource_sha,
@@ -47,7 +58,7 @@ module EasyML
           )
         end
 
-        EasyML::Column.import(columns, on_duplicate_key_update: { columns: %i[
+        EasyML::Column.import(all_columns, on_duplicate_key_update: { columns: %i[
                                          statistics
                                          learned_at
                                          sample_values

data/app/models/easy_ml/dataset.rb

@@ -362,15 +362,13 @@ module EasyML
           update(workflow_status: "analyzing")
           fully_reload
           yield
-        ensure
-          unlock!
         end
       rescue => e
         update(workflow_status: "failed")
-        e.backtrace.grep(/easy_ml/).each do |line|
-          puts line
-        end
+        EasyML::Event.handle_error(self, e)
         raise e
+      ensure
+        unlock!
       end
     end
 
@@ -797,8 +795,7 @@ module EasyML
       df = df.clone
       df = apply_features(df)
       processed.save(:train, df)
-      learn(delete: false)
-      learn_statistics(type: :processed, computed: true)
+      learn_statistics(type: :processed)
       processed.cleanup
     end
 
@@ -900,5 +897,21 @@ module EasyML
     def underscored_name
       name.gsub(/\s{2,}/, " ").gsub(/\s/, "_").downcase
     end
+
+    TIME_METHODS = %w(
+      refresh
+      prepare_features
+      refresh_datasource
+      split_data
+      fit
+      normalize_all
+      learn
+      learn_statistics
+      fit_features
+    )
+    include EasyML::Timing
+    TIME_METHODS.each do |method|
+      measure_method_timing method
+    end
   end
 end

data/app/models/easy_ml/event.rb

@@ -57,11 +57,26 @@ module EasyML
     end
 
     def self.easy_ml_context(stacktrace)
-      stacktrace.select { |loc| loc.match?(/easy_ml/) }
+      stacktrace.select(&MATCH_EASY_ML_CONTEXT)
+    end
+
+    MATCH_USER_CONTEXT = proc { |loc| !loc.match?(/features|evaluators/) && !loc.match?(/easy_ml/) }
+    MATCH_EASY_ML_CONTEXT = proc { |loc| loc.match?(/easy_ml/) }
+
+    def self.user_context?(stacktrace)
+      stacktrace.any?(&MATCH_USER_CONTEXT)
+    end
+
+    def self.get_context(stacktrace)
+      if user_context?(stacktrace)
+        stacktrace
+      else
+        easy_ml_context(stacktrace)
+      end
     end
 
     def self.called_by?(matcher)
-      easy_ml_context(caller).any? { |line| line.match?(matcher) }
+      get_context(caller).any? { |line| line.match?(matcher) }
     end
 
     def self.format_stacktrace(error)
@@ -69,7 +84,7 @@ module EasyML
 
       topline = error.inspect
 
-      stacktrace = easy_ml_context(error.backtrace)
+      stacktrace = get_context(error.backtrace)
 
       %(#{topline}
 

data/app/models/easy_ml/feature.rb

@@ -250,12 +250,14 @@ module EasyML
       if async && job_count > 1
         EasyML::ComputeFeatureJob.enqueue_ordered_batches(jobs)
       else
-        jobs.each do |feature_batch|
+        feature_idx = ordered_features.index_by(&:id)
+        change_set = jobs.map do |feature_batch|
           feature_batch.each do |batch_args|
             EasyML::ComputeFeatureJob.perform(nil, batch_args)
           end
-          feature = EasyML::Feature.find(feature_batch.first.dig(:feature_id))
+          feature = feature_idx[feature_batch.first.dig(:feature_id)]
           feature.after_fit
+          feature
         end
         dataset.after_fit_features
       end
@@ -380,11 +382,8 @@ module EasyML
 
       df_len_was = df.shape[0]
       orig_df = df.clone
-      begin
-        result = adapter.transform(df, self)
-      rescue => e
-        raise "Feature #{feature_class}#transform failed: #{e.message}"
-      end
+      result = adapter.transform(df, self)
+
       raise "Feature '#{name}' must return a Polars::DataFrame, got #{result.class}" unless result.is_a?(Polars::DataFrame)
       df_len_now = result.shape[0]
       missing_columns = orig_df.columns - result.columns

data/app/models/easy_ml/lineage.rb

@@ -18,8 +18,8 @@ module EasyML
       def learn(column)
         @lineage = EasyML::Column::Lineage.new(column).lineage
 
-        existing_lineage = where(column_id: column.id)
-        missing_lineage = @lineage.select { |l| !existing_lineage.exists?(key: l[:key]) }
+        existing_lineage = column.lineages.index_by(&:key)
+        missing_lineage = @lineage.select { |l| !existing_lineage.key?(l[:key].to_s) }
 
         missing_lineage = missing_lineage.map { |l|
           EasyML::Lineage.new(
@@ -29,7 +29,7 @@ module EasyML
             description: l[:description],
           )
         }
-        existing_lineage = existing_lineage.map do |lineage|
+        existing_lineage = existing_lineage.map do |key, lineage|
           matching_lineage = @lineage.detect { |ll| ll[:key].to_sym == lineage.key.to_sym }
 
           lineage&.assign_attributes(

data/app/models/easy_ml/model.rb

@@ -556,13 +556,15 @@ module EasyML
       self.sha = model_file.sha
       save
       dataset.upload_remote_files
-      snapshot.tap do
-        # Prepare the model to be retrained (reset values so they don't conflict with our snapshotted version)
-        bump_version(force: true)
-        dataset.bump_versions(version)
-        self.model_file = new_model_file!
-        save
-      end
+      model_snapshot = snapshot
+
+      # Prepare the model to be retrained (reset values so they don't conflict with our snapshotted version)
+      bump_version(force: true)
+      dataset.bump_versions(version)
+      self.model_file = new_model_file!
+      save
+
+      model_snapshot
     end
 
     def cannot_deploy_reasons

data/app/models/easy_ml/pca_model.rb

@@ -11,7 +11,10 @@
 module EasyML
   class PCAModel < ActiveRecord::Base
     def model
-      Marshal.load(read_attribute(:model))
+      model = read_attribute(:model)
+      return nil if model.nil?
+
+      Marshal.load(model)
     end
 
     def model=(model)

data/lib/easy_ml/railtie/templates/migration/add_unique_constraint_to_dataset_names.rb.tt

@@ -4,10 +4,5 @@ class AddUniqueConstraintToDatasetNames < ActiveRecord::Migration[<%= ActiveReco
       remove_index :easy_ml_datasets, :name
     end
     add_index :easy_ml_datasets, :name, unique: true
-
-    if index_exists?(:easy_ml_dataset_histories, :name)
-      remove_index :easy_ml_dataset_histories, :name
-    end
-    add_index :easy_ml_dataset_histories, :name, unique: true
   end
 end

data/lib/easy_ml/timing.rb

@@ -19,9 +19,7 @@ module EasyML
         result = send(method_alias, *args, **kwargs, &block)
         ending = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         elapsed = ending - starting
-        10.times do
-          puts "#{method_name} took #{elapsed.round(2)} seconds"
-        end
+        puts "#{method_name} took #{elapsed} seconds"
         # StatsD.measure("#{Rails.env}.#{prefix.present? ? "#{prefix}." : ""}#{method_name}.timing", elapsed)
         result
       end

data/lib/easy_ml/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module EasyML
-  VERSION = "0.2.0-rc91"
+  VERSION = "0.2.0-rc93"
 
   module Version
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: easy_ml
 version: !ruby/object:Gem::Version
-  version: 0.2.0.pre.rc91
+  version: 0.2.0.pre.rc93
 platform: ruby
 authors:
 - Brett Shollenberger
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-03-04 00:00:00.000000000 Z
+date: 2025-03-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord