easy_ml 0.2.0.pre.rc90 → 0.2.0.pre.rc92

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '08be0b67dba395b4aa3493a0a0fa6e5cde31246f299a14460590c8ced298d557'
-  data.tar.gz: d72734f3d5045e1f3554eecadb641d141fbb89f02b3c6f0cb265e5588c6d2866
+  metadata.gz: 428c2f3e0ce95c3f41ea3e8c31a8cdbc3709a9bd4332d0736b6b2b4000677430
+  data.tar.gz: 2bad8c01bbb1a65f3135765b815c88b3ac405a48e9a97d30173f44ac5755c4b2
 SHA512:
-  metadata.gz: a873a1cf9b00fd84dc0912392f4de8140eff18377c97e3d9464c7ce2d73a3a82c16f0ae34d09b095be290ca6cb287bc28bfa76864d1e003db99d308bc44413da
-  data.tar.gz: 47bc5ff93e92e8d51b62dad043917ce3b110a4141aa9e404456f2934dbe4b5d198506af81af7ddda78b511b96a564abc6df62834e376b02a4d7a6859dafdecfc
+  metadata.gz: '08218be1a52ee2f871da922b176af2ca6555739af0bec44065bfc81dfdfcb84b7a27c3501b66915b5d07db4bce2427ee4324e38f30a3534564c1be3df1d991a2'
+  data.tar.gz: 2eb3d84ec7fc7be829ad27bf689c617905299110cc40d0185a81d94c3b165c2039168b36473806140c220a4cb7f91c91a905d611657bfc8205ff2f77c257edb9

data/app/models/easy_ml/column.rb

@@ -515,24 +515,24 @@ module EasyML
     end
 
     def cast_statement(df, df_col, expected_dtype)
-      expected_dtype = expected_dtype.class
+      expected_dtype = expected_dtype.is_a?(Polars::DataType) ? expected_dtype : expected_dtype.class
       actual_type = df[df_col].dtype
 
       cast_statement = case expected_dtype
-      when Polars::Boolean
-        case actual_type
-        when Polars::Boolean
-          Polars.col(df_col).cast(expected_dtype)
-        when Polars::String, Polars::Categorical
-          Polars.col(df_col).eq("true").cast(expected_dtype)
-        when Polars::Null
-          Polars.col(df_col)
-        else
-          raise "Unexpected dtype: #{actual_type} for column: #{df_col}"
-        end
-      else
-        Polars.col(df_col).cast(expected_dtype)
-      end
+                       when Polars::Boolean
+                          case actual_type
+                          when Polars::Boolean
+                            Polars.col(df_col).cast(expected_dtype)
+                          when Polars::String, Polars::Categorical
+                            Polars.col(df_col).eq("true").cast(expected_dtype)
+                          when Polars::Null
+                            Polars.col(df_col)
+                          else
+                            raise "Unexpected dtype: #{actual_type} for column: #{df_col}"
+                          end
+                        else
+                          Polars.col(df_col).cast(expected_dtype)
+                        end
 
       cast_statement.alias(df_col)
     end

data/app/models/easy_ml/column_list.rb

@@ -104,7 +104,11 @@ module EasyML
         expected_dtype = schema[df_col.to_sym]
         db_col.cast_statement(df, df_col, expected_dtype)
       end
-      df = df.with_columns(cast_statements)
+      begin
+        df = df.with_columns(cast_statements)
+      rescue => e
+        binding.pry
+      end
     end
 
     def cast(processed_or_raw)

data/app/models/easy_ml/dataset.rb

@@ -217,6 +217,7 @@ module EasyML
       @processed = processed.cp(version)
       save.tap do
         features.each(&:bump_version)
+        EasyML::Feature.import(features.to_a, on_duplicate_key_update: [:version])
       end
     end
 
@@ -633,11 +634,16 @@ module EasyML
     def apply_missing_columns(df, inference: false)
       return df unless inference
 
-      missing_columns = (col_order(inference: inference) - df.columns).compact.uniq
+      required_cols = col_order(inference: inference).compact.uniq
       columns.one_hots.each do |one_hot|
-        missing_columns -= one_hot.virtual_columns
-        missing_columns += [one_hot.name]
+        virtual_columns = one_hot.virtual_columns
+        if virtual_columns.all? { |vc| df.columns.include?(vc) }
+          required_cols -= virtual_columns
+        else
+          required_cols += [one_hot.name]
+        end
       end
+      missing_columns = required_cols - df.columns
       df.with_columns(missing_columns.map { |f| Polars.lit(nil).alias(f) })
     end
 
@@ -894,5 +900,21 @@ module EasyML
     def underscored_name
       name.gsub(/\s{2,}/, " ").gsub(/\s/, "_").downcase
     end
+
+    TIME_METHODS = %w(
+      refresh
+      prepare_features
+      refresh_datasource
+      split_data
+      fit
+      normalize_all
+      learn
+      learn_statistics
+      fit_features
+    )
+    include EasyML::Timing
+    TIME_METHODS.each do |method|
+      measure_method_timing method
+    end
   end
 end

data/lib/easy_ml/data/dataset_manager/writer/partitioned.rb

@@ -17,9 +17,7 @@ module EasyML
           end
 
           def wipe
-            folders.each do |folder|
-              FileUtils.rm_rf(File.join(root_dir, folder))
-            end
+            super
             clear_all_keys
           end
 

data/lib/easy_ml/railtie/generators/migration/migration_generator.rb

@@ -59,6 +59,7 @@ module EasyML
             add_pca_model_id_to_easy_ml_columns
             add_workflow_status_to_easy_ml_dataset_histories
             add_metadata_to_easy_ml_predictions
+            add_unique_constraint_to_dataset_names
           ].freeze
 
           # Specify the next migration number

data/lib/easy_ml/railtie/templates/migration/add_unique_constraint_to_dataset_names.rb.tt

@@ -0,0 +1,8 @@
+class AddUniqueConstraintToDatasetNames < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    if index_exists?(:easy_ml_datasets, :name)
+      remove_index :easy_ml_datasets, :name
+    end
+    add_index :easy_ml_datasets, :name, unique: true
+  end
+end

data/lib/easy_ml/timing.rb

@@ -19,9 +19,7 @@ module EasyML
         result = send(method_alias, *args, **kwargs, &block)
         ending = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         elapsed = ending - starting
-        10.times do
-          puts "#{method_name} took #{elapsed.round(2)} seconds"
-        end
+        puts "#{method_name} took #{elapsed} seconds"
         # StatsD.measure("#{Rails.env}.#{prefix.present? ? "#{prefix}." : ""}#{method_name}.timing", elapsed)
         result
       end

data/lib/easy_ml/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module EasyML
-  VERSION = "0.2.0-rc90"
+  VERSION = "0.2.0-rc92"
 
   module Version
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: easy_ml
 version: !ruby/object:Gem::Version
-  version: 0.2.0.pre.rc90
+  version: 0.2.0.pre.rc92
 platform: ruby
 authors:
 - Brett Shollenberger
@@ -803,6 +803,7 @@ files:
 - lib/easy_ml/railtie/templates/migration/add_raw_schema_to_datasets.rb.tt
 - lib/easy_ml/railtie/templates/migration/add_sha_to_datasources_datasets_and_columns.rb.tt
 - lib/easy_ml/railtie/templates/migration/add_slug_to_easy_ml_models.rb.tt
+- lib/easy_ml/railtie/templates/migration/add_unique_constraint_to_dataset_names.rb.tt
 - lib/easy_ml/railtie/templates/migration/add_unique_constraint_to_easy_ml_model_names.rb.tt
 - lib/easy_ml/railtie/templates/migration/add_workflow_status_to_easy_ml_dataset_histories.rb.tt
 - lib/easy_ml/railtie/templates/migration/add_workflow_status_to_easy_ml_features.rb.tt