easy_ml 0.2.0.pre.rc72 → 0.2.0.pre.rc75

data/app/models/easy_ml/models/xgboost.rb

@@ -199,7 +199,7 @@ module EasyML
         set_default_wandb_project_name unless tuning
 
         # Prepare validation data
-        x_valid, y_valid = dataset.valid(split_ys: true)
+        x_valid, y_valid = dataset.valid(split_ys: true, select: dataset.col_order)
         d_valid = preprocess(x_valid, y_valid)
 
         num_iterations = hyperparameters.to_h[:n_estimators]
@@ -217,7 +217,7 @@ module EasyML
         callbacks << ::XGBoost::EvaluationMonitor.new(period: 1)
 
         # Generate batches without loading full dataset
-        batches = dataset.train(split_ys: true, batch_size: batch_size, batch_start: batch_start, batch_key: batch_key)
+        batches = dataset.train(split_ys: true, batch_size: batch_size, batch_start: batch_start, batch_key: batch_key, select: dataset.col_order)
         prev_xs = []
         prev_ys = []
 
@@ -281,9 +281,32 @@ module EasyML
         return @booster
       end
 
-      def weights
-        @booster.save_model("tmp/xgboost_model.json")
-        @booster.get_dump
+      def weights(model_file)
+        return nil unless model_file.present? && model_file.fit?
+
+        JSON.parse(model_file.read)
+      end
+
+      def set_weights(model_file, weights)
+        raise ArgumentError, "Weights must be provided" unless weights.present?
+
+        # Create a temp file with the weights
+        temp_file = Tempfile.new(["xgboost_weights", ".json"])
+        begin
+          temp_file.write(weights.to_json)
+          temp_file.close
+
+          # Load the weights into a new booster
+          initialize_model do
+            attrs = {
+              params: hyperparameters.to_h.symbolize_keys.compact,
+              model_file: temp_file.path,
+            }.compact
+            booster_class.new(**attrs)
+          end
+        ensure
+          temp_file.unlink
+        end
       end
 
       def predict(xs)
@@ -397,11 +420,12 @@ module EasyML
 
       def prepare_data
         if @d_train.nil?
-          x_sample, y_sample = dataset.train(split_ys: true, limit: 5)
+          col_order = dataset.col_order
+          x_sample, y_sample = dataset.train(split_ys: true, limit: 5, select: col_order)
           preprocess(x_sample, y_sample) # Ensure we fail fast if the dataset is misconfigured
-          x_train, y_train = dataset.train(split_ys: true)
-          x_valid, y_valid = dataset.valid(split_ys: true)
-          x_test, y_test = dataset.test(split_ys: true)
+          x_train, y_train = dataset.train(split_ys: true, select: col_order)
+          x_valid, y_valid = dataset.valid(split_ys: true, select: col_order)
+          x_test, y_test = dataset.test(split_ys: true, select: col_order)
           @d_train = preprocess(x_train, y_train)
           @d_valid = preprocess(x_valid, y_valid)
           @d_test = preprocess(x_test, y_test)

data/app/models/easy_ml/retraining_job.rb

@@ -6,7 +6,6 @@
 #  model_id         :bigint
 #  frequency        :string           not null
 #  at               :json             not null
-#  evaluator        :json
 #  tuning_enabled   :boolean          default(FALSE)
 #  tuner_config     :json
 #  tuning_frequency :string
@@ -160,6 +159,14 @@ module EasyML
       }[frequency.to_sym]
     end
 
+    def to_config
+      EasyML::Export::RetrainingJob.to_config(self)
+    end
+
+    def self.from_config(config, model)
+      EasyML::Import::RetrainingJob.from_config(config, model)
+    end
+
     private
 
     def metric_class

data/app/models/easy_ml/retraining_run.rb

@@ -83,7 +83,6 @@ module EasyML
             completed_at: failed_reasons.none? ? Time.current : nil,
             error_message: failed_reasons.any? ? failed_reasons&.first : nil,
             model: training_model,
-            metrics: training_model.evaluate,
             best_params: best_params,
             tuner_job_id: tuner&.id,
             metadata: tuner&.metadata,
@@ -109,6 +108,7 @@ module EasyML
         end
         true
       rescue => e
+        puts EasyML::Event.easy_ml_context(e.backtrace)
         EasyML::Event.handle_error(self, e)
         update!(
           status: "failed",
@@ -150,14 +150,15 @@ module EasyML
 
       training_model.dataset.refresh
       evaluator = retraining_job.evaluator.symbolize_keys
-      x_true, y_true = training_model.dataset.test(split_ys: true)
-      y_pred = training_model.predict(x_true)
+      x_test, y_test = training_model.dataset.test(split_ys: true)
+      y_pred = training_model.predict(x_test)
 
       metric = evaluator[:metric].to_sym
       metrics = EasyML::Core::ModelEvaluator.evaluate(
         model: training_model,
         y_pred: y_pred,
-        y_true: y_true,
+        y_true: y_test,
+        x_true: x_test,
         dataset: training_model.dataset.test(all_columns: true),
         evaluator: evaluator,
       )
@@ -176,6 +177,7 @@ module EasyML
 
       {
         metric_value: metric_value,
+        metrics: metrics,
         threshold: threshold,
         threshold_direction: threshold_direction,
         deployable: deployable,

data/app/models/easy_ml/splitter.rb

@@ -75,6 +75,14 @@ module EasyML
       }
     end
 
+    def to_config
+      EasyML::Export::Splitter.to_config(self)
+    end
+
+    def self.from_config(config, dataset)
+      EasyML::Import::Splitter.from_config(config, dataset)
+    end
+
     def split(df, &block)
       adapter.split(df, &block)
     end

data/app/models/lineage_history.rb

@@ -0,0 +1,6 @@
+module EasyML
+  class LineageHistory < ActiveRecord::Base
+    self.table_name = "easy_ml_lineage_histories"
+    include Historiographer::History
+  end
+end

data/app/serializers/easy_ml/column_serializer.rb

@@ -28,10 +28,16 @@ module EasyML
 
     attributes :id, :name, :description, :dataset_id, :datatype, :polars_datatype, :preprocessing_steps,
                :hidden, :drop_if_null, :sample_values, :statistics, :is_target,
-               :is_computed, :computed_by, :lineage
+               :is_computed, :computed_by
 
     attribute :required do |object|
       object.required?
     end
+
+    attribute :lineage do |column|
+      column.lineages.map do |lineage|
+        LineageSerializer.new(lineage).serializable_hash.dig(:data, :attributes)
+      end
+    end
   end
 end

data/app/serializers/easy_ml/dataset_serializer.rb

@@ -59,7 +59,8 @@ module EasyML
     end
 
     attribute :columns do |dataset|
-      dataset.columns.order(:id).map do |column|
+      col_order = dataset.col_order
+      dataset.columns.sort_by { |c| col_order.index(c.name) || Float::INFINITY }.map do |column|
         ColumnSerializer.new(column).serializable_hash.dig(:data, :attributes)
       end
     end

data/app/serializers/easy_ml/lineage_serializer.rb

@@ -0,0 +1,9 @@
+require "jsonapi/serializer"
+
+module EasyML
+  class LineageSerializer
+    include JSONAPI::Serializer
+
+    attributes :id, :key, :description, :occurred_at
+  end
+end

data/config/routes.rb

@@ -18,10 +18,15 @@ EasyML::Engine.routes.draw do
     member do
       post :train
       post :abort
+      get :download
+      post :upload
       get :retraining_runs, to: "retraining_runs#index"
     end
+    collection do
+      get "new", as: "new"
+      post :upload
+    end
     resources :deploys, only: [:create]
-    get "new", on: :collection, as: "new"
   end
 
   resources :retraining_runs, only: [:show]
@@ -30,6 +35,7 @@ EasyML::Engine.routes.draw do
   resources :datasources, as: :easy_ml_datasources do
     member do
       post :sync
+      post :abort
     end
   end
 
@@ -37,6 +43,12 @@ EasyML::Engine.routes.draw do
   resources :datasets, as: :easy_ml_datasets do
     member do
       post :refresh
+      post :abort
+      get :download
+      post :upload
+    end
+    collection do
+      post :upload
     end
   end
 

data/lib/easy_ml/core/tuner/adapters/base_adapter.rb

@@ -4,7 +4,7 @@ module EasyML
       module Adapters
         class BaseAdapter
           attr_accessor :config, :project_name, :tune_started_at, :model,
-                        :x_true, :y_true, :metadata, :model
+                        :x_valid, :y_valid, :metadata, :model
 
           def initialize(options = {})
             @model = options[:model]
@@ -12,8 +12,8 @@ module EasyML
             @project_name = options[:project_name]
             @tune_started_at = options[:tune_started_at]
             @model = options[:model]
-            @x_true = options[:x_true]
-            @y_true = options[:y_true]
+            @x_valid = options[:x_valid]
+            @y_valid = options[:y_valid]
             @metadata = options[:metadata] || {}
           end
 

data/lib/easy_ml/core/tuner.rb

@@ -6,7 +6,7 @@ module EasyML
     class Tuner
       attr_accessor :model, :dataset, :project_name, :task, :config,
                     :metrics, :objective, :n_trials, :direction, :evaluator,
-                    :study, :results, :adapter, :tune_started_at, :x_true, :y_true,
+                    :study, :results, :adapter, :tune_started_at, :x_valid, :y_valid,
                     :project_name, :job, :current_run, :trial_enumerator, :progress_block,
                     :tuner_job, :dataset
 
@@ -34,7 +34,7 @@ module EasyML
             config: config,
             project_name: project_name,
             tune_started_at: nil,  # This will be set during tune
-            y_true: nil, # This will be set during tune
+            y_valid: nil, # This will be set during tune
           )
         end
       end
@@ -70,17 +70,16 @@ module EasyML
         @job = tuner_job
         @study = Optuna::Study.new(direction: direction)
         @results = []
-        model.evaluator = evaluator if evaluator.present?
         model.task = task
 
         model.dataset.refresh if model.dataset.needs_refresh?
-        x_true, y_true = model.dataset.test(split_ys: true)
-        self.x_true = x_true
-        self.y_true = y_true
-        self.dataset = model.dataset.test(all_columns: true)
+        x_valid, y_valid = model.dataset.valid(split_ys: true, select: model.dataset.col_order)
+        self.x_valid = x_valid
+        self.y_valid = y_valid
+        self.dataset = model.dataset.valid(all_columns: true)
         adapter.tune_started_at = tune_started_at
-        adapter.y_true = y_true
-        adapter.x_true = x_true
+        adapter.x_valid = x_valid
+        adapter.y_valid = y_valid
 
         model.prepare_data unless model.batch_mode
         model.prepare_callbacks(self)
@@ -99,6 +98,7 @@ module EasyML
             @results.push(result)
             @study.tell(@current_trial, result)
           rescue StandardError => e
+            puts EasyML::Event.easy_ml_context(e.backtrace)
             @tuner_run.update!(status: :failed, hyperparameters: {})
             puts "Optuna failed with: #{e.message}"
             raise e
@@ -118,6 +118,7 @@ module EasyML
 
         best_run&.hyperparameters
       rescue StandardError => e
+        puts EasyML::Event.easy_ml_context(e.backtrace)
         tuner_job&.update!(status: :failed, completed_at: Time.current)
         raise e
       end
@@ -137,9 +138,9 @@ module EasyML
           end
         end
 
-        y_pred = model.predict(x_true)
+        y_pred = model.predict(x_valid)
         model.metrics = metrics
-        metrics = model.evaluate(y_pred: y_pred, y_true: y_true, x_true: x_true, dataset: dataset)
+        metrics = model.evaluate(y_pred: y_pred, y_true: y_valid, x_true: x_valid, dataset: dataset)
         metric = metrics.symbolize_keys.dig(model.evaluator[:metric].to_sym)
 
         puts metrics

data/lib/easy_ml/data/polars_column.rb

@@ -2,7 +2,7 @@ require_relative "date_converter"
 
 module EasyML
   module Data
-    module PolarsColumn
+    class PolarsColumn
       TYPE_MAP = {
         float: Polars::Float64,
         integer: Polars::Int64,
@@ -14,132 +14,181 @@ module EasyML
         categorical: Polars::Categorical,
         null: Polars::Null,
       }
-      POLARS_MAP = TYPE_MAP.invert.stringify_keys
+      POLARS_MAP = {
+        Polars::Float64 => :float,
+        Polars::Int64 => :integer,
+        Polars::Float32 => :float,
+        Polars::Int32 => :integer,
+        Polars::Boolean => :boolean,
+        Polars::Datetime => :datetime,
+        Polars::Date => :date,
+        Polars::String => :string,
+        Polars::Categorical => :categorical,
+        Polars::Null => :null,
+      }.stringify_keys
+      include EasyML::Timing
+
       class << self
         def polars_to_sym(polars_type)
-          POLARS_MAP.dig(polars_type.class.to_s)
+          new.polars_to_sym(polars_type)
+        end
+
+        def determine_type(series, polars_type = false)
+          new.determine_type(series, polars_type)
         end
 
         def parse_polars_dtype(dtype_string)
-          case dtype_string
-          when /^Polars::Datetime/
-            time_unit = dtype_string[/time_unit: "(.*?)"/, 1]
-            time_zone = dtype_string[/time_zone: (.*)?\)/, 1]
-            time_zone = time_zone == "nil" ? nil : time_zone&.delete('"')
-            Polars::Datetime.new(time_unit, time_zone)
-          when /^Polars::/
-            Polars.const_get(dtype_string.split("::").last)
-          else
-            raise ArgumentError, "Unknown Polars data type: #{dtype_string}"
-          end
+          new.parse_polars_dtype(dtype_string)
+        end
+
+        def get_polars_type(dtype)
+          new.get_polars_type(dtype)
+        end
+
+        def polars_dtype_to_sym(dtype_string)
+          new.polars_dtype_to_sym(dtype_string)
         end
 
         def sym_to_polars(symbol)
-          TYPE_MAP.dig(symbol)
+          new.sym_to_polars(symbol)
         end
+      end
 
-        # Determines the semantic type of a field based on its data
-        # @param series [Polars::Series] The series to analyze
-        # @return [Symbol] One of :numeric, :datetime, :categorical, or :text
-        def determine_type(series, polars_type = false)
-          dtype = series.dtype
-
-          if dtype.is_a?(Polars::Utf8)
-            string_type = determine_string_type(series)
-            if string_type == :datetime
-              date = EasyML::Data::DateConverter.maybe_convert_date(series)
-              return polars_type ? date[date.columns.first].dtype : :datetime
-            end
-          end
+      def polars_to_sym(polars_type)
+        return nil if polars_type.nil?
+
+        if polars_type.is_a?(Polars::DataType)
+          POLARS_MAP.dig(polars_type.class.to_s)
+        else
+          polars_type.to_sym if TYPE_MAP.keys.include?(polars_type.to_sym)
+        end
+      end
 
-          type_name = case dtype
-            when Polars::Float64
-              :float
-            when Polars::Int64
-              :integer
-            when Polars::Datetime
-              :datetime
-            when Polars::Date
-              :date
-            when Polars::Boolean
-              :boolean
-            when Polars::Utf8
-              determine_string_type(series)
-            when Polars::Null
-              :null
-            else
-              :categorical
-            end
-
-          polars_type ? sym_to_polars(type_name) : type_name
+      def parse_polars_dtype(dtype_string)
+        case dtype_string
+        when /^Polars::Datetime/
+          time_unit = dtype_string[/time_unit: "(.*?)"/, 1]
+          time_zone = dtype_string[/time_zone: (.*)?\)/, 1]
+          time_zone = time_zone == "nil" ? nil : time_zone&.delete('"')
+          Polars::Datetime.new(time_unit, time_zone)
+        when /^Polars::/
+          Polars.const_get(dtype_string.split("::").last)
+        else
+          raise ArgumentError, "Unknown Polars data type: #{dtype_string}"
         end
+      end
 
-        # Determines if a string field is a date, text, or categorical
-        # @param series [Polars::Series] The string series to analyze
-        # @return [Symbol] One of :datetime, :text, or :categorical
-        def determine_string_type(series)
-          if EasyML::Data::DateConverter.maybe_convert_date(Polars::DataFrame.new({ temp: series }),
-                                                            :temp)[:temp].dtype.is_a?(Polars::Datetime)
+      def sym_to_polars(symbol)
+        TYPE_MAP.dig(symbol.to_sym)
+      end
+
+      # Determines the semantic type of a field based on its data
+      # @param series [Polars::Series] The series to analyze
+      # @return [Symbol] One of :numeric, :datetime, :categorical, or :text
+      def determine_type(series, polars_type = false)
+        dtype = series.dtype
+
+        if dtype.is_a?(Polars::Utf8)
+          string_type = determine_string_type(series)
+          if string_type == :datetime
+            date = EasyML::Data::DateConverter.maybe_convert_date(series)
+            return polars_type ? date[date.columns.first].dtype : :datetime
+          end
+        end
+
+        type_name = case dtype
+          when Polars::Float64
+            :float
+          when Polars::Int64
+            :integer
+          when Polars::Datetime
             :datetime
+          when Polars::Date
+            :date
+          when Polars::Boolean
+            :boolean
+          when Polars::Utf8
+            determine_string_type(series)
+          when Polars::Null
+            :null
           else
-            categorical_or_text?(series)
+            :categorical
           end
+
+        polars_type ? sym_to_polars(type_name) : type_name
+      end
+
+      measure_method_timing :determine_type
+
+      # Determines if a string field is a date, text, or categorical
+      # @param series [Polars::Series] The string series to analyze
+      # @return [Symbol] One of :datetime, :text, or :categorical
+      def determine_string_type(series)
+        if EasyML::Data::DateConverter.maybe_convert_date(Polars::DataFrame.new({ temp: series }),
+                                                          :temp)[:temp].dtype.is_a?(Polars::Datetime)
+          :datetime
+        else
+          categorical_or_text?(series)
         end
+      end
 
-        # Determines if a string field is categorical or free text
-        # @param series [Polars::Series] The string series to analyze
-        # @return [Symbol] Either :categorical or :text
-        def categorical_or_text?(series)
-          return :categorical if series.null_count == series.len
+      measure_method_timing :determine_string_type
 
-          # Get non-null count for percentage calculations
-          non_null_count = series.len - series.null_count
-          return :categorical if non_null_count == 0
+      # Determines if a string field is categorical or free text
+      # @param series [Polars::Series] The string series to analyze
+      # @return [Symbol] Either :categorical or :text
+      def categorical_or_text?(series)
+        return :categorical if series.null_count == series.len
 
-          # Get value counts as percentages
-          value_counts = series.value_counts(parallel: true)
-          percentages = value_counts.with_column(
-            (value_counts["count"] / non_null_count.to_f * 100).alias("percentage")
-          )
+        # Get non-null count for percentage calculations
+        non_null_count = series.len - series.null_count
+        return :categorical if non_null_count == 0
 
-          # Check if any category represents more than 10% of the data
-          max_percentage = percentages["percentage"].max
-          return :text if max_percentage < 10.0
+        # Get value counts as percentages
+        value_counts = series.value_counts(parallel: true)
+        percentages = value_counts.with_column(
+          (value_counts["count"] / non_null_count.to_f * 100).alias("percentage")
+        )
 
-          # Calculate average percentage per category
-          avg_percentage = 100.0 / series.n_unique
+        # Check if any category represents more than 10% of the data
+        max_percentage = percentages["percentage"].max
+        return :text if max_percentage < 10.0
 
-          # If average category represents less than 1% of data, it's likely text
-          avg_percentage < 1.0 ? :text : :categorical
-        end
+        # Calculate average percentage per category
+        avg_percentage = 100.0 / series.n_unique
 
-        # Returns whether the field type is numeric
-        # @param field_type [Symbol] The field type to check
-        # @return [Boolean]
-        def numeric?(field_type)
-          field_type == :numeric
-        end
+        # If average category represents less than 1% of data, it's likely text
+        avg_percentage < 1.0 ? :text : :categorical
+      end
 
-        # Returns whether the field type is categorical
-        # @param field_type [Symbol] The field type to check
-        # @return [Boolean]
-        def categorical?(field_type)
-          field_type == :categorical
-        end
+      measure_method_timing :categorical_or_text?
 
-        # Returns whether the field type is datetime
-        # @param field_type [Symbol] The field type to check
-        # @return [Boolean]
-        def datetime?(field_type)
-          field_type == :datetime
-        end
+      # Returns whether the field type is numeric
+      # @param field_type [Symbol] The field type to check
+      # @return [Boolean]
+      def numeric?(field_type)
+        field_type == :numeric
+      end
 
-        # Returns whether the field type is text
-        # @param field_type [Symbol] The field type to check
-        # @return [Boolean]
-        def text?(field_type)
-          field_type == :text
-        end
+      # Returns whether the field type is categorical
+      # @param field_type [Symbol] The field type to check
+      # @return [Boolean]
+      def categorical?(field_type)
+        field_type == :categorical
+      end
+
+      # Returns whether the field type is datetime
+      # @param field_type [Symbol] The field type to check
+      # @return [Boolean]
+      def datetime?(field_type)
+        field_type == :datetime
+      end
+
+      # Returns whether the field type is text
+      # @param field_type [Symbol] The field type to check
+      # @return [Boolean]
+      def text?(field_type)
+        field_type == :text
       end
     end
   end

data/lib/easy_ml/data/polars_reader.rb

@@ -88,17 +88,20 @@ module EasyML
       end
 
       def query(files = nil, drop_cols: [], filter: nil, limit: nil, select: nil, unique: nil, sort: nil, descending: false,
-                             batch_size: nil, batch_start: nil, batch_key: nil, &block)
+                             batch_size: nil, batch_start: nil, batch_key: nil, lazy: false, &block)
         files ||= self.files
         PolarsReader.query(files, drop_cols: drop_cols, filter: filter, limit: limit,
                                   select: select, unique: unique, sort: sort, descending: descending,
-                                  batch_size: batch_size, batch_start: batch_start, batch_key: batch_key, &block)
+                                  batch_size: batch_size, batch_start: batch_start, batch_key: batch_key, lazy: lazy, &block)
       end
 
       def self.query(files, drop_cols: [], filter: nil, limit: nil, select: nil, unique: nil, sort: nil, descending: false,
-                            batch_size: nil, batch_start: nil, batch_key: nil, &block)
-        return query_files(files, drop_cols: drop_cols, filter: filter, limit: limit, select: select,
-                                  unique: unique, sort: sort, descending: descending).collect unless batch_size.present?
+                            batch_size: nil, batch_start: nil, batch_key: nil, lazy: false, &block)
+        unless batch_size.present?
+          result = query_files(files, drop_cols: drop_cols, filter: filter, limit: limit, select: select,
+                                      unique: unique, sort: sort, descending: descending)
+          return lazy ? result : result.collect
+        end
 
         return batch_enumerator(files, drop_cols: drop_cols, filter: filter, limit: limit, select: select, unique: unique, sort: sort, descending: descending,
                                        batch_size: batch_size, batch_start: batch_start, batch_key: batch_key) unless block_given?