easy_ml 0.2.0.pre.rc55 → 0.2.0.pre.rc57

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 53b0666a5ba25b758573564fb916da9ec5b968223e0b8164e394c6b2d176668a
-  data.tar.gz: 5180eb72880c443fd5405bba8127ab2ba9c891f51c7a7063f47862a48d10ca12
+  metadata.gz: e52412950fefc02e9b838930f132873c726440ebbc343159504d7d3287a39d05
+  data.tar.gz: 44ff18d1f1df78b542c8e536427189fce63d147e7e86623d219ed9b89c501ca7
 SHA512:
-  metadata.gz: 912c3eba2b15b1d463eb53a2e7cd689fec14c45cd58fb232a5f31114ea35f883004616977598d85b8b2e22bbd0ac19dc4e5f294878cc544007eaed1ffd61bd0d
-  data.tar.gz: b4b89abaf966fc0dc8f99e58feb80503175a97c8006ccd9d0bd17319ebdf1b1d1c615c762f159a7ea2b167abd9f0351e0cf2e4e3a7bc34694067f011a03aae0d
+  metadata.gz: 1e543781fb426a6fa7fe6ad6f5b7c924bdab38d88ac8ad7288db3a24f683661b3745a6f2176c993899a9f9737af7e54dfa59cc439a71739d3e2d2d2d75714621
+  data.tar.gz: 3f012c5a3126eec7a69c3c11dd45017f7c2ded7a2bfd5e6e70bcaa388000b19e50d19ed15dc6b47786f61b698cc081e915abade7ece544a3c8a14d0a8f5c4696

data/app/frontend/components/ScheduleModal.tsx

@@ -91,7 +91,7 @@ export function ScheduleModal({ isOpen, onClose, onSave, initialData, metrics, t
         day_of_month: initialData.retraining_job?.at?.day_of_month ?? 1
       },
       metric: initialData.retraining_job?.metric || (metrics[initialData.task]?.[0]?.value ?? ''),
-      threshold: initialData.retraining_job?.threshold || (initialData.task === 'classification' ? 0.85 : 0.1),
+      threshold: initialData.retraining_job?.threshold ?? (initialData.task === 'classification' ? 0.85 : 0.1),
       tuner_config: initialData.retraining_job?.tuner_config ? {
         n_trials: initialData.retraining_job.tuner_config.n_trials || 10,
         config: {

data/app/frontend/components/SearchableSelect.tsx

@@ -60,7 +60,10 @@ export const SearchableSelect = forwardRef<HTMLButtonElement, SearchableSelectPr
       }
     }, [isOpen]);
 
-    const handleOptionClick = (optionValue: Option['value']) => {
+    const handleOptionClick = (optionValue: Option['value'], e: React.MouseEvent) => {
+      debugger;
+      e.preventDefault();
+      e.stopPropagation();
       onChange(optionValue);
       setIsOpen(false);
       setSearchQuery('');
@@ -86,7 +89,7 @@ export const SearchableSelect = forwardRef<HTMLButtonElement, SearchableSelectPr
               placeholder="Search..."
               value={searchQuery}
               onChange={(e) => setSearchQuery(e.target.value)}
-              onClick={(e) => e.stopPropagation()}
+              onMouseDown={(e) => e.stopPropagation()}
             />
           </div>
         </div>
@@ -105,11 +108,7 @@ export const SearchableSelect = forwardRef<HTMLButtonElement, SearchableSelectPr
                     className={`w-full text-left px-4 py-2 hover:bg-gray-100 ${
                       option.value === value ? 'bg-blue-50' : ''
                     }`}
-                    onMouseDown={(e) => {
-                      e.preventDefault();
-                      e.stopPropagation();
-                      handleOptionClick(option.value);
-                    }}
+                    onMouseDown={(e) => handleOptionClick(option.value, e)}
                   >
                     <div className="flex items-center justify-between">
                       <span className="block font-medium">
@@ -140,7 +139,6 @@ export const SearchableSelect = forwardRef<HTMLButtonElement, SearchableSelectPr
           type="button"
           onMouseDown={(e) => {
             e.preventDefault();
-            e.stopPropagation();
             setIsOpen(!isOpen);
           }}
           className="w-full bg-white relative border border-gray-300 rounded-md shadow-sm pl-3 pr-10 py-2 text-left cursor-pointer focus:outline-none focus:ring-1 focus:ring-blue-500 focus:border-blue-500"

data/app/jobs/easy_ml/refresh_dataset_job.rb

@@ -3,6 +3,8 @@ module EasyML
     def perform(id)
       begin
         dataset = EasyML::Dataset.find(id)
+        return if dataset.workflow_status == :analyzing
+
         puts "Refreshing dataset #{dataset.name}"
         puts "Needs refresh? #{dataset.needs_refresh?}"
         unless dataset.needs_refresh?
@@ -12,6 +14,7 @@ module EasyML
         create_event(dataset, "started")
 
         puts "Prepare! #{dataset.name}"
+        dataset.unlock!
         dataset.prepare
         if dataset.features.needs_fit.any?
           dataset.fit_features(async: true)

data/app/models/easy_ml/column.rb

@@ -30,7 +30,6 @@ module EasyML
     validates :name, uniqueness: { scope: :dataset_id }
 
     before_save :ensure_valid_datatype
-    after_create :set_date_column_if_date_splitter
     after_save :handle_date_column_change
     before_save :set_defaults
 
@@ -41,6 +40,18 @@ module EasyML
     scope :datetime, -> { where(datatype: "datetime") }
     scope :date_column, -> { where(is_date_column: true) }
 
+    def columns
+      [name].concat(virtual_columns)
+    end
+
+    def virtual_columns
+      if one_hot?
+        allowed_categories.map { |cat| "#{name}_#{cat}" }
+      else
+        []
+      end
+    end
+
     def datatype=(dtype)
       write_attribute(:datatype, dtype)
       write_attribute(:polars_datatype, dtype)
@@ -88,9 +99,11 @@ module EasyML
     end
 
     def allowed_categories
-      return nil unless one_hot?
+      return [] unless one_hot?
+      stats = dataset.preprocessor.statistics
+      return [] if stats.nil? || stats.blank?
 
-      dataset.preprocessor.statistics.dup.to_h.dig(name.to_sym, :allowed_categories).sort.concat(["other"])
+      stats.dup.to_h.dig(name.to_sym, :allowed_categories).sort.concat(["other"])
     end
 
     def date_column?

data/app/models/easy_ml/column_list.rb

@@ -1,6 +1,6 @@
 module EasyML
   module ColumnList
-    def sync
+    def sync(delete: true)
       return unless dataset.schema.present?
 
       EasyML::Column.transaction do
@@ -8,7 +8,11 @@ module EasyML
         existing_columns = where(name: col_names)
         import_new(col_names, existing_columns)
         update_existing(existing_columns)
-        delete_missing(existing_columns)
+
+        if delete
+          delete_missing(existing_columns)
+        end
+
         if existing_columns.none? # Totally new dataset
           dataset.after_create_columns
         end

data/app/models/easy_ml/dataset.rb

@@ -175,7 +175,6 @@ module EasyML
 
     def actually_refresh
       refreshing do
-        split_data
         process_data
         fully_reload
         learn
@@ -273,10 +272,10 @@ module EasyML
       raw.split_at.present? && raw.split_at < datasource.last_updated_at
     end
 
-    def learn
+    def learn(delete: true)
       learn_schema
       learn_statistics
-      columns.sync
+      columns.sync(delete: delete)
     end
 
     def refreshing
@@ -399,7 +398,7 @@ module EasyML
 
       # Learn will update columns, so if any features have been added
       # since the last time columns were learned, we should re-learn the schema
-      learn if idx == 0 && needs_learn?(df)
+      learn(delete: false) if idx == 1 && needs_learn?(df)
       df = apply_column_mask(df, inference: inference) unless all_columns
       raise_on_nulls(df) if inference
       df, = processed.split_features_targets(df, true, target) if split_ys
@@ -516,7 +515,7 @@ module EasyML
     end
 
     def drop_cols
-      @drop_cols ||= preloaded_columns.select(&:hidden).map(&:name)
+      @drop_cols ||= preloaded_columns.select(&:hidden).flat_map(&:columns)
     end
 
     def drop_if_null

data/app/models/easy_ml/event.rb

@@ -56,14 +56,16 @@ module EasyML
       create_event(model, "failed", error)
     end
 
+    def self.easy_ml_context(stacktrace)
+      stacktrace.select { |loc| loc.match?(/easy_ml/) }
+    end
+
     def self.format_stacktrace(error)
       return nil if error.nil?
 
       topline = error.inspect
 
-      stacktrace = error.backtrace.select do |loc|
-        loc.match?(/easy_ml/)
-      end
+      stacktrace = easy_ml_context(error.backtrace)
 
       %(#{topline}
 

data/app/models/easy_ml/model.rb

@@ -354,15 +354,16 @@ module EasyML
       dataset.decode_labels(ys, col: col)
     end
 
-    def evaluate(y_pred: nil, y_true: nil, x_true: nil, evaluator: nil)
+    def evaluate(y_pred: nil, y_true: nil, x_true: nil, evaluator: nil, dataset: nil)
       evaluator ||= self.evaluator
       if y_pred.nil?
         inputs = default_evaluation_inputs
         y_pred = inputs[:y_pred]
         y_true = inputs[:y_true]
         x_true = inputs[:x_true]
+        dataset = inputs[:dataset]
       end
-      EasyML::Core::ModelEvaluator.evaluate(model: self, y_pred: y_pred, y_true: y_true, x_true: x_true, evaluator: evaluator)
+      EasyML::Core::ModelEvaluator.evaluate(model: self, y_pred: y_pred, y_true: y_true, x_true: x_true, dataset: dataset, evaluator: evaluator)
     end
 
     def evaluator
@@ -524,11 +525,13 @@ module EasyML
 
     def default_evaluation_inputs
       x_true, y_true = dataset.test(split_ys: true)
+      ds = dataset.test(all_columns: true)
       y_pred = predict(x_true)
       {
         x_true: x_true,
         y_true: y_true,
         y_pred: y_pred,
+        dataset: ds,
       }
     end
 

data/app/models/easy_ml/models/xgboost/evals_callback.rb

@@ -32,7 +32,7 @@ module EasyML
           false
         end
 
-        def validation_dataset
+        def test_dataset
           if tuner.present?
             [tuner.x_true, tuner.y_true]
           else
@@ -46,11 +46,12 @@ module EasyML
           log_frequency = 10
           if epoch % log_frequency == 0
             model.adapter.external_model = booster
-            x_true, y_true = validation_dataset
+            x_true, y_true = test_dataset
             @preprocessed ||= model.preprocess(x_true)
             y_pred = model.predict(@preprocessed)
+            dataset = model.dataset.test(all_columns: true)
 
-            metrics = model.evaluate(y_pred: y_pred, y_true: y_true, x_true: x_true)
+            metrics = model.evaluate(y_pred: y_pred, y_true: y_true, x_true: x_true, dataset: dataset)
             Wandb.log(metrics)
           end
 

data/lib/easy_ml/core/evaluators/base_evaluator.rb

@@ -32,7 +32,7 @@ module EasyML
         end
 
         # Instance methods that evaluators must implement
-        def evaluate(y_pred: nil, y_true: nil, x_true: nil)
+        def evaluate(y_pred: nil, y_true: nil, x_true: nil, dataset: nil)
           raise NotImplementedError, "#{self.class} must implement #evaluate"
         end
 

data/lib/easy_ml/core/evaluators/classification_evaluators.rb

@@ -5,7 +5,7 @@ module EasyML
         class AccuracyScore
           include BaseEvaluator
 
-          def evaluate(y_pred:, y_true:, x_true: nil)
+          def evaluate(y_pred:, y_true:, x_true: nil, dataset: nil)
             y_pred = Numo::Int32.cast(y_pred)
             y_true = Numo::Int32.cast(y_true)
             y_pred.eq(y_true).count_true.to_f / y_pred.size
@@ -23,7 +23,7 @@ module EasyML
         class PrecisionScore
           include BaseEvaluator
 
-          def evaluate(y_pred:, y_true:, x_true: nil)
+          def evaluate(y_pred:, y_true:, x_true: nil, dataset: nil)
             y_pred = Numo::Int32.cast(y_pred)
             y_true = Numo::Int32.cast(y_true)
             true_positives = (y_pred.eq(1) & y_true.eq(1)).count_true
@@ -45,7 +45,7 @@ module EasyML
         class RecallScore
           include BaseEvaluator
 
-          def evaluate(y_pred:, y_true:, x_true: nil)
+          def evaluate(y_pred:, y_true:, x_true: nil, dataset: nil)
             y_pred = Numo::Int32.cast(y_pred)
             y_true = Numo::Int32.cast(y_true)
             true_positives = (y_pred.eq(1) & y_true.eq(1)).count_true
@@ -65,9 +65,9 @@ module EasyML
         class F1Score
           include BaseEvaluator
 
-          def evaluate(y_pred:, y_true:, x_true: nil)
-            precision = PrecisionScore.new.evaluate(y_pred: y_pred, y_true: y_true)
-            recall = RecallScore.new.evaluate(y_pred: y_pred, y_true: y_true)
+          def evaluate(y_pred:, y_true:, x_true: nil, dataset: nil)
+            precision = PrecisionScore.new.evaluate(y_pred: y_pred, y_true: y_true, dataset: dataset)
+            recall = RecallScore.new.evaluate(y_pred: y_pred, y_true: y_true, dataset: dataset)
             return 0 unless (precision + recall) > 0
 
             2 * (precision * recall) / (precision + recall)
@@ -85,7 +85,7 @@ module EasyML
         class AUC
           include BaseEvaluator
 
-          def evaluate(y_pred:, y_true:, x_true: nil)
+          def evaluate(y_pred:, y_true:, x_true: nil, dataset: nil)
             y_pred = Numo::DFloat.cast(y_pred)
             y_true = Numo::Int32.cast(y_true)
 
@@ -132,8 +132,8 @@ module EasyML
         class ROC_AUC
           include BaseEvaluator
 
-          def evaluate(y_pred:, y_true:, x_true: nil)
-            AUC.new.evaluate(y_pred: y_pred, y_true: y_true)
+          def evaluate(y_pred:, y_true:, x_true: nil, dataset: nil)
+            AUC.new.evaluate(y_pred: y_pred, y_true: y_true, dataset: dataset)
           end
 
           def description

data/lib/easy_ml/core/evaluators/regression_evaluators.rb

@@ -5,7 +5,7 @@ module EasyML
         class MeanAbsoluteError
           include BaseEvaluator
 
-          def evaluate(y_pred:, y_true:, x_true: nil)
+          def evaluate(y_pred:, y_true:, x_true: nil, dataset: nil)
             (Numo::DFloat.cast(y_pred) - Numo::DFloat.cast(y_true)).abs.mean
           end
 
@@ -21,7 +21,7 @@ module EasyML
         class MeanSquaredError
           include BaseEvaluator
 
-          def evaluate(y_pred:, y_true:, x_true: nil)
+          def evaluate(y_pred:, y_true:, x_true: nil, dataset: nil)
             ((Numo::DFloat.cast(y_pred) - Numo::DFloat.cast(y_true)) ** 2).mean
           end
 
@@ -37,7 +37,7 @@ module EasyML
         class RootMeanSquaredError
           include BaseEvaluator
 
-          def evaluate(y_pred:, y_true:, x_true: nil)
+          def evaluate(y_pred:, y_true:, x_true: nil, dataset: nil)
             Math.sqrt(((Numo::DFloat.cast(y_pred) - Numo::DFloat.cast(y_true)) ** 2).mean)
           end
 
@@ -61,7 +61,7 @@ module EasyML
             "maximize"
           end
 
-          def evaluate(y_pred:, y_true:, x_true: nil)
+          def evaluate(y_pred:, y_true:, x_true: nil, dataset: nil)
             y_true = Numo::DFloat.cast(y_true)
             y_pred = Numo::DFloat.cast(y_pred)
 

data/lib/easy_ml/core/model_evaluator.rb

@@ -98,13 +98,21 @@ module EasyML
           end
         end
 
-        def evaluate(model:, y_pred:, y_true:, x_true: nil, evaluator: nil)
+        def evaluate(model:, y_pred:, y_true:, x_true: nil, evaluator: nil, dataset: nil)
           y_pred = normalize_input(y_pred)
           y_true = normalize_input(y_true)
           check_size(y_pred, y_true)
 
           metrics_results = {}
 
+          if x_true.nil?
+            x_true = model.dataset.test
+          end
+
+          if dataset.nil?
+            dataset = model.dataset.test(all_columns: true)
+          end
+
           model.metrics.each do |metric|
             evaluator_class = get(metric.to_sym)
             next unless evaluator_class
@@ -115,6 +123,7 @@ module EasyML
               y_pred: y_pred,
               y_true: y_true,
               x_true: x_true,
+              dataset: dataset,
             )
           end
 
@@ -124,7 +133,7 @@ module EasyML
             raise "Unknown evaluator: #{evaluator}" unless evaluator_class
 
             evaluator_instance = evaluator_class.new
-            response = evaluator_instance.evaluate(y_pred: y_pred, y_true: y_true, x_true: x_true)
+            response = evaluator_instance.evaluate(y_pred: y_pred, y_true: y_true, x_true: x_true, dataset: dataset)
 
             if response.is_a?(Hash)
               metrics_results.merge!(response)
@@ -145,6 +154,9 @@ module EasyML
         def normalize_input(input)
           case input
           when Array
+            if input.first.class == TrueClass || input.first.class == FalseClass
+              input = input.map { |value| value ? 1.0 : 0.0 }
+            end
             Numo::DFloat.cast(input)
           when Polars::DataFrame
             if input.columns.count > 1
@@ -152,7 +164,10 @@ module EasyML
             end
 
             normalize_input(input[input.columns.first])
-          when Polars::Series, Array
+          when Polars::Series
+            if input.dtype == Polars::Boolean
+              input = input.cast(Polars::Int64)
+            end
             Numo::DFloat.cast(input)
           else
             raise ArgumentError, "Don't know how to evaluate model with y_pred type #{input.class}"

data/lib/easy_ml/core/tuner.rb

@@ -8,7 +8,7 @@ module EasyML
                     :metrics, :objective, :n_trials, :direction, :evaluator,
                     :study, :results, :adapter, :tune_started_at, :x_true, :y_true,
                     :project_name, :job, :current_run, :trial_enumerator, :progress_block,
-                    :tuner_job
+                    :tuner_job, :dataset
 
       def initialize(options = {})
         @model = options[:model]
@@ -77,6 +77,7 @@ module EasyML
         x_true, y_true = model.dataset.test(split_ys: true)
         self.x_true = x_true
         self.y_true = y_true
+        self.dataset = model.dataset.test(all_columns: true)
         adapter.tune_started_at = tune_started_at
         adapter.y_true = y_true
         adapter.x_true = x_true
@@ -96,14 +97,6 @@ module EasyML
             run_metrics = tune_once
             result = calculate_result(run_metrics)
             @results.push(result)
-
-            params = {
-              hyperparameters: model.hyperparameters.to_h,
-              value: result,
-              status: :success,
-            }.compact
-
-            @tuner_run.update!(params)
             @study.tell(@current_trial, result)
           rescue StandardError => e
             @tuner_run.update!(status: :failed, hyperparameters: {})
@@ -138,14 +131,27 @@ module EasyML
         )
         self.current_run = @tuner_run
 
-        adapter.run_trial(@current_trial) do |model|
-          model.fit(tuning: true, &progress_block)
-          y_pred = model.predict(x_true)
-          model.metrics = metrics
-          metrics = model.evaluate(y_pred: y_pred, y_true: y_true, x_true: x_true)
-          puts metrics
-          metrics
+        model = adapter.run_trial(@current_trial) do |model|
+          model.tap do
+            model.fit(tuning: true, &progress_block)
+          end
         end
+
+        y_pred = model.predict(x_true)
+        model.metrics = metrics
+        metrics = model.evaluate(y_pred: y_pred, y_true: y_true, x_true: x_true, dataset: dataset)
+        metric = metrics.symbolize_keys.dig(model.evaluator[:metric].to_sym)
+
+        puts metrics
+
+        params = {
+          hyperparameters: model.hyperparameters.to_h,
+          value: metric,
+          status: :success,
+        }.compact
+
+        @tuner_run.update!(params)
+        metrics
       end
 
       private

data/lib/easy_ml/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module EasyML
-  VERSION = "0.2.0-rc55"
+  VERSION = "0.2.0-rc57"
 
   module Version
   end

data/public/easy_ml/assets/.vite/manifest.json

@@ -1,6 +1,6 @@
 {
   "entrypoints/Application.tsx": {
-    "file": "assets/entrypoints/Application.tsx-Dr3jVR78.js",
+    "file": "assets/entrypoints/Application.tsx-DTZ2348z.js",
     "name": "entrypoints/Application.tsx",
     "src": "entrypoints/Application.tsx",
     "isEntry": true,