easy_ml 0.2.0.pre.rc72 → 0.2.0.pre.rc75

data/app/models/easy_ml/import/dataset.rb

@@ -0,0 +1,148 @@
+module EasyML
+  module Import
+    class Dataset
+      def self.permitted_keys
+        @permitted_keys ||= EasyML::Dataset.columns.map(&:name).map(&:to_sym) -
+                            EasyML::Export::Dataset::UNCONFIGURABLE_COLUMNS.map(&:to_sym) +
+                            [:columns, :features, :splitter, :datasource]
+      end
+
+      def self.from_config(json_config, action: nil, dataset: nil)
+        raise ArgumentError, "Target dataset must be specified" if action == :update && dataset.nil?
+
+        config = json_config.is_a?(String) ? JSON.parse(json_config) : json_config
+        dataset_config = config["dataset"]
+
+        # Extract configs for related models
+        datasource_config = dataset_config.delete("datasource")
+        splitter_config = dataset_config.delete("splitter")
+        columns_config = dataset_config.delete("columns") || []
+        features_config = dataset_config.delete("features") || []
+
+        if action == :create
+          name = dataset_config["name"]
+          dataset = EasyML::Dataset.find_by(name: name)
+          action = dataset.present? ? :update : :create
+        end
+        raise ArgumentError, "Action must be specified" unless action.present?
+
+        if action == :create
+          create_dataset(
+            dataset_config,
+            datasource_config,
+            splitter_config,
+            columns_config,
+            features_config
+          )
+        elsif action == :update
+          update_dataset(
+            dataset,
+            dataset_config,
+            columns_config,
+            features_config
+          )
+        else
+          raise ArgumentError, "Invalid action: #{action}. Must be :create or :update"
+        end
+      end
+
+      private
+
+      def self.create_dataset(dataset_config, datasource_config, splitter_config, columns_config, features_config)
+        # Create new datasource
+        datasource = EasyML::Datasource.find_or_create_by(name: datasource_config["name"]) do |ds|
+          ds.assign_attributes(datasource_config)
+        end
+        datasource.update!(datasource_config)
+
+        # Create new dataset
+        dataset = EasyML::Dataset.create!(
+          dataset_config.merge(datasource: datasource)
+        )
+
+        # Create splitter if config exists
+        EasyML::Splitter.from_config(splitter_config, dataset) if splitter_config.present?
+
+        # Create columns
+        columns_config.each do |column_config|
+          EasyML::Column.from_config(column_config, dataset, action: :create)
+        end
+
+        # Create features
+        features_config.each do |feature_config|
+          EasyML::Feature.from_config(feature_config, dataset, action: :create)
+        end
+
+        dataset
+      end
+
+      def self.update_dataset(dataset, dataset_config, columns_config, features_config)
+        # Update dataset attributes except name (preserve original name)
+        dataset.update!(dataset_config.except("name", "datasource"))
+
+        needs_refresh = false
+
+        # Update existing columns
+        columns_config.each do |column_config|
+          column_name = column_config["name"]
+          existing_column = dataset.columns.find_by(name: column_name)
+          
+          if existing_column
+            old_drop_if_null = existing_column.drop_if_null
+            new_drop_if_null = column_config["drop_if_null"]
+            
+            # Check if drop_if_null has changed
+            needs_refresh ||= !new_drop_if_null.nil? && old_drop_if_null != new_drop_if_null
+          end
+
+          EasyML::Column.from_config(column_config, dataset, action: :update)
+        end
+
+        # Update or create features
+        features_config.each do |feature_config|
+          EasyML::Feature.from_config(feature_config, dataset, action: :update)
+        end
+
+        # Refresh if needed
+        dataset.refresh_async if needs_refresh
+
+        dataset
+      end
+
+      def self.validate(dataset_config)
+        extra_keys = dataset_config.keys.map(&:to_sym) - permitted_keys
+        raise ArgumentError, "Invalid dataset keys: #{extra_keys.join(", ")}" unless extra_keys.empty?
+
+        if dataset_config[:splitter].present?
+          dataset_config[:splitter] = EasyML::Import::Splitter.validate(dataset_config[:splitter])
+        end
+
+        if dataset_config[:columns].present?
+          unless dataset_config[:columns].is_a?(Array)
+            raise ArgumentError, "Columns configuration must be an array"
+          end
+          dataset_config[:columns].each_with_index do |col_config, idx|
+            unless col_config.is_a?(Hash)
+              raise ArgumentError, "Each column configuration must be a hash, at index #{idx}"
+            end
+            EasyML::Import::Column.validate(col_config, idx)
+          end
+        end
+
+        if dataset_config[:features].present?
+          unless dataset_config[:features].is_a?(Array)
+            raise ArgumentError, "Features configuration must be an array"
+          end
+          dataset_config[:features].each_with_index do |feat_config, idx|
+            unless feat_config.is_a?(Hash)
+              raise ArgumentError, "Each feature configuration must be a hash, at index #{idx}"
+            end
+            EasyML::Import::Feature.validate(feat_config, idx)
+          end
+        end
+
+        dataset_config
+      end
+    end
+  end
+end

data/app/models/easy_ml/import/feature.rb

@@ -0,0 +1,36 @@
+module EasyML
+  module Import
+    class Feature
+      def self.permitted_keys
+        @permitted_keys ||= EasyML::Feature.columns.map(&:name).map(&:to_sym) -
+                            EasyML::Export::Feature::UNCONFIGURABLE_COLUMNS.map(&:to_sym)
+      end
+
+      def self.from_config(config, dataset, action: :create)
+        feature_name = config["name"]
+        existing_feature = dataset.features.find_by(name: feature_name)
+
+        case action
+        when :create
+          dataset.features.create!(config)
+        when :update
+          if existing_feature
+            existing_feature.update!(config)
+            existing_feature
+          else
+            # Features can be added during update, unlike columns
+            dataset.features.create!(config)
+          end
+        else
+          raise ArgumentError, "Invalid action: #{action}. Must be :create or :update"
+        end
+      end
+
+      def self.validate(config, idx)
+        extra_keys = config.keys.map(&:to_sym) - permitted_keys
+        raise ArgumentError, "Invalid keys in feature config at index #{idx}: #{extra_keys.join(", ")}" unless extra_keys.empty?
+        config
+      end
+    end
+  end
+end

data/app/models/easy_ml/import/model.rb

@@ -0,0 +1,136 @@
+module EasyML
+  module Import
+    class Model
+      def self.permitted_keys
+        @permitted_keys ||= EasyML::Model.columns.map(&:name).map(&:to_sym) -
+                            EasyML::Export::Model::UNCONFIGURABLE_COLUMNS.map(&:to_sym) +
+                            [:weights] +
+                            EasyML::Model.configuration_attributes.map(&:to_sym) +
+                            [:dataset, :splitter, :retraining_job]
+      end
+
+      def self.from_config(json_config, action: nil, model: nil, include_dataset: true, dataset: nil)
+        raise ArgumentError, "Action must be specified" unless action.present?
+        raise ArgumentError, "Target model must be specified" if action == :update && model.nil?
+        raise ArgumentError, "Dataset must be specified when creating a model" if action == :create && !include_dataset && dataset.nil?
+
+        config = json_config.is_a?(String) ? JSON.parse(json_config) : json_config
+        config = config.deep_dup.with_indifferent_access
+
+        # Validate the configuration
+        validate(config)
+        model_config = config["model"]
+
+        # Config variables would skip custom setters, so better to manually merge
+        configuration = model_config.delete("configuration")
+        model_config.merge!(configuration) if configuration.present?
+
+        case action
+        when :create
+          create_model(model_config, include_dataset: include_dataset, dataset: dataset)
+        when :update
+          update_model(model, model_config, include_dataset: include_dataset)
+        else
+          raise ArgumentError, "Invalid action: #{action}. Must be :create or :update"
+        end
+      end
+
+      private
+
+      def self.create_model(model_config, include_dataset:, dataset:)
+        # Handle dataset if included
+        model_dataset = if include_dataset && model_config["dataset"].present?
+            dataset_config = { "dataset" => model_config.delete("dataset") }
+            EasyML::Import::Dataset.from_config(dataset_config, action: :create)
+          else
+            dataset
+          end
+
+        # Create model
+        model = EasyML::Model.new(model_config.except("weights", "dataset", "retraining_job"))
+        model.dataset = model_dataset
+
+        model_name = model_config["name"]
+        if (existing_model = EasyML::Model.find_by(name: model_name)).present?
+          model.name = generate_unique_name(model_name)
+        end
+        model.save!
+
+        if model_config["retraining_job"].present?
+          retraining_job = EasyML::RetrainingJob.from_config(model_config["retraining_job"], model)
+          model.retraining_job = retraining_job
+          model.save!
+          model.reload
+        end
+
+        # Update weights if present
+        if model_config["weights"].present?
+          model.update!(weights: model_config["weights"])
+          model.import
+        end
+
+        model
+      end
+
+      def self.update_model(model, model_config, include_dataset:)
+        # Update dataset if included
+        if include_dataset && model_config["dataset"].present?
+          dataset_config = { "dataset" => model_config.delete("dataset") }
+          EasyML::Import::Dataset.from_config(dataset_config, action: :update, dataset: model.dataset)
+        end
+
+        # Update model attributes except name (preserve original name)
+        model.update!(model_config.except("name", "weights", "dataset", "retraining_job"))
+
+        if model_config["retraining_job"].present?
+          retraining_job = EasyML::RetrainingJob.from_config(model_config["retraining_job"], model)
+          model.retraining_job = retraining_job
+          model.save!
+          model.reload
+        end
+
+        # Update weights if present
+        if model_config["weights"].present?
+          model.update!(weights: model_config["weights"])
+          model.import
+        end
+
+        model
+      end
+
+      def self.validate(json_config)
+        config = json_config.is_a?(String) ? JSON.parse(json_config) : json_config
+        config = config.deep_dup.with_indifferent_access
+
+        # Validate root keys: must have only "model"
+        extra_keys = config.keys.map(&:to_sym) - [:model]
+        raise ArgumentError, "Invalid root keys: #{extra_keys.join(", ")}" unless extra_keys.empty?
+
+        model_config = config[:model]
+        # Validate that model_config does not contain keys that are unconfigurable
+        extra_keys = model_config.keys.map(&:to_sym) - permitted_keys
+        raise ArgumentError, "Invalid model keys: #{extra_keys.join(", ")}" unless extra_keys.empty?
+
+        # Delegate nested validations to individual importers
+        if model_config["dataset"].present?
+          model_config["dataset"] = EasyML::Import::Dataset.validate(model_config["dataset"])
+        end
+
+        if model_config["retraining_job"].present?
+          model_config["retraining_job"] = EasyML::Import::RetrainingJob.validate(model_config["retraining_job"])
+        end
+
+        config
+      end
+
+      def self.generate_unique_name(base_name)
+        revision = EasyML::Model.where("name LIKE ?", "#{base_name} (Revision %)")
+          .map { |m| m.name.match(/\(Revision (\d+)\)/).try(:[], 1).try(:to_i) }
+          .compact
+          .max || 0
+
+        "#{base_name} (Revision #{revision + 1})"
+      end
+    end
+  end
+end

data/app/models/easy_ml/import/retraining_job.rb

@@ -0,0 +1,29 @@
+module EasyML
+  module Import
+    class RetrainingJob
+      def self.permitted_keys
+        @permitted_keys ||= EasyML::RetrainingJob.columns.map(&:name).map(&:to_sym) -
+                            EasyML::Export::RetrainingJob::UNCONFIGURABLE_COLUMNS.map(&:to_sym)
+      end
+
+      def self.from_config(config, model)
+        existing_job = model.get_retraining_job
+        existing_job.update!(config)
+        existing_job
+      end
+
+      def self.validate(config)
+        return nil unless config.present?
+
+        unless config.is_a?(Hash)
+          raise ArgumentError, "Retraining job configuration must be a hash"
+        end
+
+        extra_keys = config.keys.map(&:to_sym) - permitted_keys
+        raise ArgumentError, "Invalid retraining job keys: #{extra_keys.join(", ")}" unless extra_keys.empty?
+
+        config
+      end
+    end
+  end
+end

data/app/models/easy_ml/import/splitter.rb

@@ -0,0 +1,34 @@
+module EasyML
+  module Import
+    class Splitter
+      def self.permitted_keys
+        @permitted_keys ||= EasyML::Splitter.columns.map(&:name).map(&:to_sym) -
+                            EasyML::Export::Splitter::UNCONFIGURABLE_COLUMNS.map(&:to_sym)
+      end
+
+      def self.from_config(config, dataset)
+        return nil unless config.present?
+
+        if dataset.splitter.present?
+          dataset.splitter.update!(config)
+          dataset.splitter
+        else
+          dataset.create_splitter!(config)
+        end
+      end
+
+      def self.validate(config)
+        return nil unless config.present?
+
+        unless config.is_a?(Hash)
+          raise ArgumentError, "Splitter configuration must be a hash"
+        end
+
+        extra_keys = config.keys.map(&:to_sym) - permitted_keys
+        raise ArgumentError, "Invalid splitter keys: #{extra_keys.join(", ")}" unless extra_keys.empty?
+
+        config
+      end
+    end
+  end
+end

data/app/models/easy_ml/lineage.rb

@@ -0,0 +1,44 @@
+# == Schema Information
+#
+# Table name: easy_ml_lineages
+#
+#  id          :bigint           not null, primary key
+#  column_id   :bigint           not null
+#  key         :string           not null
+#  description :string
+#  occurred_at :datetime
+#  created_at  :datetime         not null
+#  updated_at  :datetime         not null
+#
+module EasyML
+  class Lineage < ActiveRecord::Base
+    belongs_to :column
+
+    class << self
+      def learn(column)
+        @lineage = EasyML::Column::Lineage.new(column).lineage
+
+        existing_lineage = where(column_id: column.id)
+        missing_lineage = @lineage.select { |l| !existing_lineage.exists?(key: l[:key]) }
+
+        missing_lineage = missing_lineage.map { |l|
+          EasyML::Lineage.new(
+            column_id: column.id,
+            key: l[:key],
+            occurred_at: l[:occurred_at],
+            description: l[:description],
+          )
+        }
+        existing_lineage = existing_lineage.map do |lineage|
+          matching_lineage = @lineage.detect { |ll| ll[:key].to_sym == lineage.key.to_sym }
+
+          lineage&.assign_attributes(
+            occurred_at: matching_lineage[:occurred_at],
+            description: matching_lineage[:description],
+          )
+        end
+        missing_lineage.concat(existing_lineage)
+      end
+    end
+  end
+end

data/app/models/easy_ml/model.rb

@@ -45,7 +45,7 @@ module EasyML
     MODEL_NAMES = MODEL_OPTIONS.keys.freeze
     MODEL_CONSTANTS = MODEL_OPTIONS.values.map(&:constantize)
 
-    add_configuration_attributes :task, :objective, :hyperparameters, :evaluator, :callbacks, :metrics
+    add_configuration_attributes :task, :objective, :hyperparameters, :callbacks, :metrics
     MODEL_CONSTANTS.flat_map(&:configuration_attributes).each do |attribute|
       add_configuration_attributes attribute
     end
@@ -53,10 +53,10 @@ module EasyML
     belongs_to :dataset
     belongs_to :model_file, class_name: "EasyML::ModelFile", foreign_key: "model_file_id", optional: true
 
-    has_one :retraining_job, class_name: "EasyML::RetrainingJob"
+    has_one :retraining_job, class_name: "EasyML::RetrainingJob", dependent: :destroy
     accepts_nested_attributes_for :retraining_job
-    has_many :retraining_runs, class_name: "EasyML::RetrainingRun"
-    has_many :deploys, class_name: "EasyML::Deploy"
+    has_many :retraining_runs, class_name: "EasyML::RetrainingRun", dependent: :destroy
+    has_many :deploys, class_name: "EasyML::Deploy", dependent: :destroy
 
     scope :deployed, -> { EasyML::ModelHistory.deployed }
 
@@ -127,26 +127,41 @@ module EasyML
       end
     end
 
+    def trained?
+      retraining_runs.where(status: :success).exists?
+    end
+
+    def deployed?
+      inference_version.present?
+    end
+
+    def weights=(weights)
+      raise ArgumentError, "Cannot set weights on model without type" unless model_type.present?
+
+      model_file = get_model_file
+      adapter.set_weights(model_file, weights)
+      save_model_file
+    end
+
+    def weights
+      adapter.weights(get_model_file)
+    end
+
     def get_retraining_job
-      if retraining_job
-        self.evaluator = retraining_job.evaluator
-        evaluator = self.evaluator.symbolize_keys
-      else
-        default_eval = Core::ModelEvaluator.default_evaluator(task)
-        self.evaluator = default_eval
-        evaluator = default_eval
-      end
+      return retraining_job if retraining_job.present?
 
-      retraining_job || create_retraining_job(
-        model: self,
-        active: false,
-        evaluator: evaluator,
-        metric: evaluator[:metric],
-        direction: evaluator[:direction],
-        threshold: evaluator[:threshold],
-        frequency: "month",
-        at: { hour: 0, day_of_month: 1 },
-      )
+      evaluator = Core::ModelEvaluator.default_evaluator(task).symbolize_keys
+
+      method = persisted? ? :create_retraining_job : :build_retraining_job
+
+      send(method,
+           model: self,
+           active: false,
+           metric: evaluator[:metric],
+           direction: evaluator[:direction],
+           threshold: evaluator[:threshold],
+           frequency: "month",
+           at: { hour: 0, day_of_month: 1 })
     end
 
     def pending_run
@@ -154,6 +169,15 @@ module EasyML
       job.retraining_runs.find_or_create_by(status: "pending", model: self)
     end
 
+    def import
+      lock_model do
+        run = pending_run
+        run.wrap_training do
+          [self, hyperparameters.to_h]
+        end
+      end
+    end
+
     def actually_train(&progress_block)
       lock_model do
         run = pending_run
@@ -193,6 +217,20 @@ module EasyML
       "training:#{self.name}:#{self.id}"
     end
 
+    def hyperparameters=(hyperparameters)
+      return unless model_type.present?
+
+      @hypers = adapter.build_hyperparameters(hyperparameters)
+    end
+
+    def hyperparameters
+      @hypers ||= adapter.build_hyperparameters(@hyperparameters)
+    end
+
+    def callbacks
+      @cbs ||= adapter.build_callbacks(@callbacks)
+    end
+
     def hyperparameter_search(&progress_block)
       tuner = retraining_job.tuner_config.symbolize_keys
       extra_params = {
@@ -239,16 +277,11 @@ module EasyML
     alias_method :latest_version, :inference_version
     alias_method :deployed, :inference_version
 
-    def hyperparameters
-      @hypers ||= adapter.build_hyperparameters(@hyperparameters)
-    end
-
-    def callbacks
-      @cbs ||= adapter.build_callbacks(@callbacks)
-    end
-
     def predict(xs)
       load_model!
+      unless xs.is_a?(XGBoost::DMatrix)
+        xs = dataset.normalize(xs, inference: true)
+      end
       adapter.predict(xs)
     end
 
@@ -361,6 +394,10 @@ module EasyML
       dataset.decode_labels(ys, col: col)
     end
 
+    def evaluator
+      get_retraining_job&.evaluator || default_evaluator
+    end
+
     def evaluate(y_pred: nil, y_true: nil, x_true: nil, evaluator: nil, dataset: nil)
       evaluator ||= self.evaluator
       if y_pred.nil?
@@ -373,10 +410,6 @@ module EasyML
       EasyML::Core::ModelEvaluator.evaluate(model: self, y_pred: y_pred, y_true: y_true, x_true: x_true, dataset: dataset, evaluator: evaluator)
     end
 
-    def evaluator
-      instance_variable_get(:@evaluator) || default_evaluator
-    end
-
     def default_evaluator
       return nil unless task.present?
 
@@ -388,7 +421,7 @@ module EasyML
     end
 
     def evals
-      last_run&.metrics || {}
+      (last_run&.metrics || {}).with_indifferent_access
     end
 
     def metric_accessor(metric)
@@ -543,6 +576,28 @@ module EasyML
         end
     end
 
+    UNCONFIGURABLE_COLUMNS = %w(
+      id
+      dataset_id
+      model_file_id
+      root_dir
+      file
+      sha
+      last_trained_at
+      is_training
+      created_at
+      updated_at
+      slug
+    )
+
+    def to_config(include_dataset: false)
+      EasyML::Export::Model.to_config(self, include_dataset: include_dataset)
+    end
+
+    def self.from_config(json_config, action: nil, model: nil, include_dataset: true, dataset: nil)
+      EasyML::Import::Model.from_config(json_config, action: action, model: model, include_dataset: include_dataset, dataset: dataset)
+    end
+
     private
 
     def default_evaluation_inputs
@@ -622,6 +677,8 @@ module EasyML
     end
 
     def validate_metrics_allowed
+      set_defaults if metrics.nil? || metrics.empty?
+
       unknown_metrics = metrics.select { |metric| allowed_metrics.exclude?(metric) }
       return unless unknown_metrics.any?
 
@@ -631,7 +688,7 @@ module EasyML
 
     def set_slug
       if slug.nil? && name.present?
-        self.slug = name.gsub(/\s/, "_").downcase
+        self.slug = name.gsub(/\s/, "_").gsub(/[^a-zA-Z0-9_]/, "").downcase
       end
     end
   end

data/app/models/easy_ml/model_file.rb

@@ -97,5 +97,11 @@ module EasyML
     def extension_allowlist
       %w[bin model json]
     end
+
+    def write(content)
+      FileUtils.mkdir_p(File.dirname(full_path))
+      File.write(full_path, content)
+      upload(full_path)
+    end
   end
 end

data/app/models/easy_ml/models/xgboost/evals_callback.rb

@@ -32,9 +32,9 @@ module EasyML
           false
         end
 
-        def test_dataset
+        def valid_dataset
           if tuner.present?
-            [tuner.x_true, tuner.y_true]
+            [tuner.x_valid, tuner.y_valid]
           else
             model.dataset.valid(split_ys: true)
           end
@@ -46,12 +46,12 @@ module EasyML
           log_frequency = 10
           if epoch % log_frequency == 0
             model.adapter.external_model = booster
-            x_true, y_true = test_dataset
-            @preprocessed ||= model.preprocess(x_true)
+            x_valid, y_valid = valid_dataset
+            @preprocessed ||= model.preprocess(x_valid)
             y_pred = model.predict(@preprocessed)
-            dataset = model.dataset.test(all_columns: true)
+            dataset = model.dataset.valid(all_columns: true)
 
-            metrics = model.evaluate(y_pred: y_pred, y_true: y_true, x_true: x_true, dataset: dataset)
+            metrics = model.evaluate(y_pred: y_pred, y_true: y_valid, x_true: x_valid, dataset: dataset)
             Wandb.log(metrics)
           end
 
@@ -67,7 +67,7 @@ module EasyML
         def after_training(booster)
           return booster unless wandb_enabled?
 
-          if model.last_run&.wandb_url.nil?
+          if model.last_run.present? && model.last_run&.wandb_url.nil?
             if tuner.present? && !tuner.current_run.wandb_url.present?
               tuner.current_run.wandb_url = Wandb.current_run.url
             end