easy_ml 0.2.0.pre.rc85 → 0.2.0.pre.rc88

data/app/frontend/pages/DatasetsPage.tsx

@@ -15,7 +15,6 @@ interface Props {
 const ITEMS_PER_PAGE = 6;
 
 export default function DatasetsPage({ datasets }: Props) {
-  console.log(`change`)
   const { rootPath } = usePage().props;
   const [searchQuery, setSearchQuery] = useState('');
   const [currentPage, setCurrentPage] = useState(1);

data/app/frontend/types/dataset.ts

@@ -37,6 +37,7 @@ export type PreprocessingStep = {
     | "categorical"
     | "constant"
     | "today";
+  encoding?: "one_hot" | "ordinal" | "embedding" | null;
   params: {
     value?: number;
     constant?: string;
@@ -45,8 +46,10 @@ export type PreprocessingStep = {
       min?: number;
       max?: number;
     };
-    one_hot?: boolean;
-    ordinal_encoding?: boolean;
+    llm?: string;
+    model?: string;
+    dimensions?: number;
+    preset?: string;
   };
 };
 

data/app/models/easy_ml/column/imputers/base.rb

@@ -15,6 +15,12 @@ module EasyML
             Imputers.methods_by_class[self] << m.to_sym
           end
 
+          def encoding_applies(e)
+            Imputers.supported_encodings << e.to_sym
+            Imputers.encodings_by_class[self] ||= []
+            Imputers.encodings_by_class[self] << e.to_sym
+          end
+
           def description
             "Unknown preprocessing method"
           end
@@ -32,7 +38,7 @@ module EasyML
         end
 
         def applies?
-          method_applies? || param_applies?
+          method_applies? || param_applies? || encoding_applies?
         end
 
         def method_applies?
@@ -43,6 +49,12 @@ module EasyML
           params.keys.any? { |p| imputers_own_params.include?(p.to_sym) && params[p] != false }
         end
 
+        def encoding_applies?
+          return false unless encoding.present?
+
+          imputers_own_encodings.include?(encoding.to_sym)
+        end
+
         def imputers_own_methods
           Imputers.methods_by_class[self.class] || []
         end
@@ -51,6 +63,10 @@ module EasyML
           Imputers.params_by_class[self.class] || []
         end
 
+        def imputers_own_encodings
+          Imputers.encodings_by_class[self.class] || []
+        end
+
         def params
           @preprocessing_step.dig(:params)
         end
@@ -59,6 +75,10 @@ module EasyML
           @preprocessing_step.dig(:method)
         end
 
+        def encoding
+          @preprocessing_step.dig(:encoding)
+        end
+
         def statistics(*args)
           if column.is_computed
             column.statistics.dig(:processed, *args)
@@ -74,8 +94,9 @@ module EasyML
         def inspect
           params_str = params ? params.map { |k, v| "#{k}: #{v}" }.join(", ") : "none"
           method_str = method ? method : "none"
+          encoding_str = encoding ? encoding : "none"
 
-          "#<#{self.class.name} method=#{method_str.inspect} params={#{params_str}}>"
+          "#<#{self.class.name} method=#{method_str.inspect} encoding=#{encoding_str.inspect} params={#{params_str}}>"
         end
 
         alias_method :to_s, :inspect

data/app/models/easy_ml/column/imputers/embedding_encoder.rb

@@ -0,0 +1,18 @@
+module EasyML
+  class Column
+    class Imputers
+      class EmbeddingEncoder < Base
+        encoding_applies :embedding
+
+        def self.description
+          "Generate embeddings"
+        end
+
+        def transform(df)
+          df = column.embed(df)
+          df
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/imputers/imputer.rb

@@ -31,6 +31,7 @@ module EasyML
             Today,
             OneHotEncoder,
             OrdinalEncoder,
+            EmbeddingEncoder,
           ]
         end
 

data/app/models/easy_ml/column/imputers/most_frequent.rb

@@ -13,7 +13,7 @@ module EasyML
 
           most_frequent = statistics(:most_frequent_value)
           df = df.with_column(
-            Polars.col(column.name).fill_null(most_frequent).alias(column.name)
+            Polars.col(column.name).fill_null(Polars.lit(most_frequent).cast(column.polars_datatype)).alias(column.name)
           )
           df
         end

data/app/models/easy_ml/column/imputers/one_hot_encoder.rb

@@ -2,7 +2,7 @@ module EasyML
   class Column
     class Imputers
       class OneHotEncoder < Base
-        param_applies :one_hot
+        encoding_applies :one_hot
 
         def self.description
           "One-hot encoder"

data/app/models/easy_ml/column/imputers/ordinal_encoder.rb

@@ -2,7 +2,7 @@ module EasyML
   class Column
     class Imputers
       class OrdinalEncoder < Base
-        param_applies :ordinal_encoding
+        encoding_applies :ordinal
 
         def self.description
           "Ordinal encoder"

data/app/models/easy_ml/column/imputers.rb

@@ -5,56 +5,50 @@ module EasyML
 
       ALLOWED_PARAMS = {
         constant: [:constant],
-        categorical: %i[categorical_min one_hot ordinal_encoding],
-        most_frequent: %i[one_hot ordinal_encoding],
+        categorical: %i[categorical_min],
+        most_frequent: [],
+        embedding: %i[llm model preset dimensions],
         mean: [:clip],
         median: [:clip],
       }
 
+      LABELS = {
+        ffill: "Forward Fill",
+        categorical: "Categorical",
+        mean: "Mean",
+        median: "Median",
+        constant: "Constant Value",
+        most_frequent: "Most Frequent",
+        today: "Current Date",
+      }
+
       PREPROCESSING_STRATEGIES = {
-        float: [
-          { value: "ffill", label: "Forward Fill" },
-          { value: "mean", label: "Mean" },
-          { value: "median", label: "Median" },
-          { value: "constant", label: "Constant Value" },
-        ],
-        integer: [
-          { value: "ffill", label: "Forward Fill" },
-          { value: "mean", label: "Mean" },
-          { value: "median", label: "Median" },
-          { value: "constant", label: "Constant Value" },
-        ],
-        boolean: [
-          { value: "ffill", label: "Forward Fill" },
-          { value: "most_frequent", label: "Most Frequent" },
-          { value: "constant", label: "Constant Value" },
-        ],
-        datetime: [
-          { value: "ffill", label: "Forward Fill" },
-          { value: "constant", label: "Constant Value" },
-          { value: "today", label: "Current Date" },
-        ],
-        string: [
-          { value: "ffill", label: "Forward Fill" },
-          { value: "most_frequent", label: "Most Frequent" },
-          { value: "constant", label: "Constant Value" },
-        ],
-        text: [
-          { value: "ffill", label: "Forward Fill" },
-          { value: "most_frequent", label: "Most Frequent" },
-          { value: "constant", label: "Constant Value" },
-        ],
-        categorical: [
-          { value: "ffill", label: "Forward Fill" },
-          { value: "categorical", label: "Categorical" },
-          { value: "most_frequent", label: "Most Frequent" },
-          { value: "constant", label: "Constant Value" },
-        ],
-      }.freeze
+        float: %w(most_frequent ffill mean median constant),
+        integer: %w(most_frequent ffill mean median constant),
+        boolean: %w(ffill most_frequent constant),
+        datetime: %w(ffill today constant),
+        string: %w(ffill most_frequent constant),
+        text: %w(ffill most_frequent constant),
+        categorical: %w(ffill categorical most_frequent constant),
+      }.transform_values do |strategies|
+        strategies.map do |strategy|
+          {
+            value: strategy,
+            label: LABELS[strategy.to_sym],
+          }
+        end
+      end
+
+      ENCODING_STRATEGIES = {
+        categorical: %w(embedding one_hot ordinal),
+        string: %w(embedding),
+        text: %w(embedding),
+      }
 
       def self.constants
         {
           preprocessing_strategies: PREPROCESSING_STRATEGIES,
+          encoding_strategies: ENCODING_STRATEGIES,
         }
       end
 
@@ -66,6 +60,10 @@ module EasyML
         @methods_by_class ||= {}
       end
 
+      def self.encodings_by_class
+        @encodings_by_class ||= {}
+      end
+
       def self.supported_params
         @supported_params ||= []
       end
@@ -74,6 +72,10 @@ module EasyML
         @supported_methods ||= []
       end
 
+      def self.supported_encodings
+        @supported_encodings ||= []
+      end
+
       def initialize(column, imputers: [])
         @column = column
         @dataset = column.dataset
@@ -88,6 +90,10 @@ module EasyML
         def supported_methods
           @supported_methods ||= []
         end
+
+        def supported_encodings
+          @supported_encodings ||= []
+        end
       end
 
       def imputers

data/app/models/easy_ml/column/selector.rb

@@ -62,8 +62,8 @@ module EasyML
           kwargs[:select] = []
         end
 
-        if (selected == :processed || (selected.nil? && !dataset.needs_refresh?)) && column.one_hot?
-          kwargs[:select] << column.virtual_columns
+        if (selected == :processed || (selected.nil? && !dataset.needs_refresh?)) && column.has_virtual_columns?
+          kwargs[:select] << column.aliases
         else
           kwargs[:select] << column.name
         end