easy_ml 0.2.0.pre.rc102 → 0.2.0.pre.rc104

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4c4736c4959fd2d08faac5dbe0c4525014edb8faa7e5b914875a0a84f58e53f2
-  data.tar.gz: bbab12ed80cf8c3bd608388648cd8362d7f4b46408b135aaf79ef494dca7deed
+  metadata.gz: 617edee53d32c1340b1a996a48e6a8a60d8cccff4345e27b2e5cf2ffc926c4ac
+  data.tar.gz: 225c133b9365d62e579e39e862ef76efaf7759d1c40a59b89968441381c8c5ee
 SHA512:
-  metadata.gz: 8dd7645d2b4da2d03a0c3fc1eaf9bcfdfd05ae31e9871782154ade2149ca4269ee5d78e6cb959d6f10e498cdbd427dfb958dc37c3b1208b0fe8885abac61dcad
-  data.tar.gz: 6690b85ba40db78063ffe8fbf3b9302b82a52c3137f5a59d8ba8bb70838ebd931965968185b6031b90fa0b9d8f3192bcc1d2150e3cce43a5ad1959af738d180e
+  metadata.gz: 8a31abca5a4086eab323b1dfc112c05f89682bbe1fab0211ed111a0978cdc43f45dfd3ece03c6920001cfb4286d22b41e52421d280f02476e2feb63f0214eefd
+  data.tar.gz: 1f109b18e911eee6f81fe797f80320cf37a06e5ae25e4eb76894ffa98dbc931a66b63264462d36ce58e29ab0a7b95e90d522457e53b4fc15a71ece5473d73389

data/app/frontend/components/dataset/PreprocessingConfig.tsx

@@ -1028,7 +1028,6 @@ export function PreprocessingConfig({
                       label: strategy.label
                     })) || [])
                   ]}
-                  options={constants.preprocessing_strategies[selectedType]}
                 />
 
                 {renderStrategySpecificInfo('training')}

data/app/models/easy_ml/column.rb

@@ -522,27 +522,44 @@ module EasyML
       EasyML::Import::Column.from_config(config, dataset, action: action)
     end
 
-    def cast_statement(df, df_col, expected_dtype)
-      expected_dtype = expected_dtype.is_a?(Polars::DataType) ? expected_dtype.class : expected_dtype
-      actual_type = df[df_col].dtype
+    def cast_statement(series = nil)
+      expected_dtype = polars_datatype
+      actual_type = series&.dtype || expected_dtype
+
+      return Polars.col(name).cast(expected_dtype).alias(name) if expected_dtype == actual_type
+
+      if encoding.present?
+        encoding_cast = case encoding.to_sym
+        when :one_hot
+          Polars.col(series.name).cast(Polars::Boolean).alias(series.name)
+        when :ordinal
+          Polars.col(series.name).cast(Polars::Int64).alias(series.name)
+        when :embedding
+          Polars.col(series.name).alias(series.name)
+        end
+        return encoding_cast
+      end
 
       cast_statement = case expected_dtype.to_s
-                       when "Polars::Boolean"
+                        when /Polars::List/
+                          # we should start tracking polars args so we can know what type of list it is
+                          Polars.col(name)
+                        when /Polars::Boolean/
                           case actual_type.to_s
-                          when "Polars::Boolean"
-                            Polars.col(df_col).cast(expected_dtype)
-                          when "Polars::Utf8", "Polars::Categorical", "Polars::String"
-                            Polars.col(df_col).eq("true").cast(expected_dtype)
-                          when "Polars::Null"
-                            Polars.col(df_col)
+                          when /Polars::Boolean/, /Polars::Int/
+                            Polars.col(name).cast(expected_dtype)
+                          when /Polars::Utf/, /Polars::Categorical/, /Polars::String/
+                            Polars.col(name).eq("true").cast(expected_dtype)
+                          when /Polars::Null/
+                            Polars.col(name)
                           else
-                            raise "Unexpected dtype: #{actual_type} for column: #{df_col}"
+                            raise "Unexpected dtype: #{actual_type} for column: #{name}"
                           end
                         else
-                          Polars.col(df_col).cast(expected_dtype)
+                          Polars.col(name).cast(expected_dtype, strict: false)
                         end
 
-      cast_statement.alias(df_col)
+      cast_statement.alias(name)
     end
 
     def cast(value)

data/app/models/easy_ml/column_list.rb

@@ -101,10 +101,10 @@ module EasyML
       end
       cast_statements = (df.columns & schema.keys.map(&:to_s)).map do |df_col|
         db_col = column_index[df_col]
-        expected_dtype = schema[df_col.to_sym]
-        db_col.cast_statement(df, df_col, expected_dtype)
+        db_col.cast_statement(df[df_col])
       end
       df = df.with_columns(cast_statements)
+      df
     end
 
     def cast(processed_or_raw)

data/app/models/easy_ml/dataset/learner/lazy/datetime.rb

@@ -10,7 +10,9 @@ module EasyML
           end
 
           def unique_count
-            Polars.col(column.name).n_unique.alias("#{column.name}__unique_count")
+            Polars.col(column.name)
+                  .cast(column.polars_datatype)
+                  .n_unique.alias("#{column.name}__unique_count")
           end
         end
       end

data/app/models/easy_ml/dataset/learner/lazy/numeric.rb

@@ -5,11 +5,30 @@ module EasyML
         class Numeric < Query
           def train_query
             super.concat([
-              Polars.col(column.name).mean.alias("#{column.name}__mean"),
-              Polars.col(column.name).median.alias("#{column.name}__median"),
-              Polars.col(column.name).min.alias("#{column.name}__min"),
-              Polars.col(column.name).max.alias("#{column.name}__max"),
-              Polars.col(column.name).std.alias("#{column.name}__std"),
+              Polars.col(column.name)
+                    .cast(column.polars_datatype)
+                    .mean
+                    .alias("#{column.name}__mean"),
+
+              Polars.col(column.name)
+                      .cast(column.polars_datatype)
+                      .median
+                      .alias("#{column.name}__median"),
+
+              Polars.col(column.name)
+                    .cast(column.polars_datatype)
+                    .min
+                    .alias("#{column.name}__min"),
+
+              Polars.col(column.name)
+                    .cast(column.polars_datatype)
+                    .max
+                    .alias("#{column.name}__max"),
+
+              Polars.col(column.name)
+                    .cast(column.polars_datatype)
+                    .std
+                    .alias("#{column.name}__std"),
             ])
           end
         end

data/app/models/easy_ml/dataset/learner/lazy/query.rb

@@ -44,25 +44,37 @@ module EasyML
           end
 
           def null_count
-            Polars.col(column.name).null_count.alias("#{column.name}__null_count")
+            Polars.col(column.name)
+                  .cast(column.polars_datatype)
+                  .null_count
+                  .alias("#{column.name}__null_count")
           end
 
           def num_rows
-            Polars.col(column.name).len.alias("#{column.name}__num_rows")
+            Polars.col(column.name)
+                  .cast(column.polars_datatype)
+                  .len
+                  .alias("#{column.name}__num_rows")
           end
 
           def most_frequent_value
-            Polars.col(column.name).filter(Polars.col(column.name).is_not_null).mode.first.alias("#{column.name}__most_frequent_value")
+            Polars.col(column.name)
+                  .cast(column.polars_datatype)
+                  .filter(Polars.col(column.name).is_not_null)
+                  .mode
+                  .first
+                  .alias("#{column.name}__most_frequent_value")
           end
 
           def last_value
             return unless dataset.date_column.present?
 
             Polars.col(column.name)
-              .sort_by(dataset.date_column.name, reverse: true, nulls_last: true)
-              .filter(Polars.col(column.name).is_not_null)
-              .first
-              .alias("#{column.name}__last_value")
+                  .cast(column.polars_datatype)
+                  .sort_by(dataset.date_column.name, reverse: true, nulls_last: true)
+                  .filter(Polars.col(column.name).is_not_null)
+                  .first
+                  .alias("#{column.name}__last_value")
           end
         end
       end

data/app/models/easy_ml/dataset/learner/lazy/string.rb

@@ -10,7 +10,10 @@ module EasyML
           end
 
           def unique_count
-            Polars.col(column.name).cast(:str).n_unique.alias("#{column.name}__unique_count")
+            Polars.col(column.name)
+                  .cast(Polars::String)
+                  .n_unique
+                  .alias("#{column.name}__unique_count")
           end
         end
       end

data/app/models/easy_ml/dataset/learner/lazy.rb

@@ -28,15 +28,15 @@ module EasyML
             )
             .select(queries).collect
           rescue => e
-            problematic_query = queries.detect { 
+            problematic_queries = queries.select { |query| 
               begin
-                dataset.send(type).send(split, all_columns: true, lazy: true).select(queries).collect
+                dataset.send(type).send(split, all_columns: true, lazy: true).select([query]).collect
                 false
               rescue => e
                 true
               end
             }
-            raise "Query failed for column #{problematic_query}, likely wrong datatype"
+            raise "Query failed for queries... likely due to wrong column datatype: #{problematic_queries.join("\n")}"
           end
         end
 
@@ -64,4 +64,4 @@ module EasyML
       end
     end
   end
-end
+end

data/lib/easy_ml/data/polars_reader.rb

@@ -175,7 +175,7 @@ module EasyML
           end
           combined_lazy_df = combined_lazy_df.with_columns(
             cast.map do |col, dtype|
-              Polars.col(col).cast(dtype).alias(col)
+              Polars.col(col).cast(dtype, strict: false).alias(col)
             end
           )
         end

data/lib/easy_ml/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module EasyML
-  VERSION = "0.2.0-rc102"
+  VERSION = "0.2.0-rc104"
 
   module Version
   end

data/public/easy_ml/assets/.vite/manifest.json

@@ -1,6 +1,6 @@
 {
   "entrypoints/Application.tsx": {
-    "file": "assets/entrypoints/Application.tsx-CRS5bRgw.js",
+    "file": "assets/entrypoints/Application.tsx-gkZ77wo8.js",
     "name": "entrypoints/Application.tsx",
     "src": "entrypoints/Application.tsx",
     "isEntry": true,