easy_ml 0.1.4 → 0.2.0.pre.rc1

data/app/models/easy_ml/retraining_job.rb

@@ -0,0 +1,278 @@
+# == Schema Information
+#
+# Table name: easy_ml_retraining_jobs
+#
+#  id               :bigint           not null, primary key
+#  model_id         :bigint
+#  frequency        :string           not null
+#  at               :json             not null
+#  evaluator        :json
+#  tuning_enabled   :boolean          default(FALSE)
+#  tuner_config     :json
+#  tuning_frequency :string
+#  last_tuning_at   :datetime
+#  active           :boolean          default(TRUE)
+#  status           :string           default("pending")
+#  last_run_at      :datetime
+#  metric           :string           not null
+#  direction        :string           not null
+#  threshold        :float            not null
+#  auto_deploy      :boolean          default(FALSE)
+#  batch_mode       :boolean
+#  batch_size       :integer
+#  batch_overlap    :integer
+#  batch_key        :string
+#  created_at       :datetime         not null
+#  updated_at       :datetime         not null
+#
+module EasyML
+  class RetrainingJob < ActiveRecord::Base
+    self.table_name = "easy_ml_retraining_jobs"
+
+    has_many :retraining_runs, class_name: "EasyML::RetrainingRun", dependent: :destroy
+    has_many :tuner_jobs, through: :retraining_runs
+
+    belongs_to :model, class_name: "EasyML::Model", inverse_of: :retraining_job
+    validates :model, presence: true,
+                      uniqueness: { message: "already has a retraining job" }
+
+    VALID_FREQUENCIES = %w[day week month always].freeze
+    FREQUENCY_TYPES = [
+      {
+        value: "day",
+        label: "Daily",
+        description: "Run once every day",
+      },
+      {
+        value: "week",
+        label: "Weekly",
+        description: "Run once every week",
+      },
+      {
+        value: "month",
+        label: "Monthly",
+        description: "Run once every month",
+      },
+    ].freeze
+    validates :frequency, presence: true, inclusion: { in: VALID_FREQUENCIES }
+    validates :metric, presence: true
+    validate :validate_metrics_allowed
+    validates :status, presence: true
+    validates :at, presence: true
+    validates :threshold, presence: true
+    validates :tuning_frequency, inclusion: {
+                                   in: VALID_FREQUENCIES,
+                                   allow_nil: true,
+                                 }
+    validate :evaluator_must_be_valid
+    validate :validate_at_format
+    after_initialize :set_direction, unless: :persisted?
+
+    scope :active, -> { joins(:model).where(active: true) }
+
+    def self.current
+      active.select do |job|
+        job.should_run?
+      end
+    end
+
+    def self.constants
+      {
+        frequency: FREQUENCY_TYPES,
+      }
+    end
+
+    def tuner_config
+      (read_attribute(:tuner_config) || {}).merge!(objective: metric).stringify_keys
+    end
+
+    def formatted_frequency
+      if active
+        FREQUENCY_TYPES.find { |type| type[:value] == frequency }[:label]
+      else
+        "Manually"
+      end
+    end
+
+    def should_run?
+      return true if last_run_at.nil?
+
+      case frequency
+      when "day"
+        current_time = Time.current
+        return false if last_run_at.to_date == current_time.to_date
+        current_time.hour == at["hour"]
+      when "week"
+        current_time = Time.current
+        return false if last_run_at.to_date >= current_time.beginning_of_week
+        current_time.wday == at["day_of_week"] && current_time.hour == at["hour"]
+      when "month"
+        current_time = Time.current
+        return false if last_run_at.to_date >= current_time.beginning_of_month
+        current_time.day == at["day_of_month"] && current_time.hour == at["hour"]
+      else
+        false
+      end
+    end
+
+    def should_tune?
+      return false unless tuning_enabled
+      return false unless tuning_frequency.present?
+      return true if last_tuning_at.nil?
+
+      case tuning_frequency
+      when "always"
+        true
+      when "hour"
+        last_tuning_at < Time.current.beginning_of_hour
+      when "day"
+        current_time = Time.current
+        current_time.hour == at["hour"] && last_tuning_at < current_time.beginning_of_day
+      when "week"
+        current_time = Time.current
+        current_time.hour == at["hour"] && current_time.wday == 0 && last_tuning_at < current_time.beginning_of_week
+      when "month"
+        current_time = Time.current
+        current_time.hour == at["hour"] && current_time.day == 1 && last_tuning_at < current_time.beginning_of_month
+      end
+    end
+
+    def metric=(metric)
+      write_attribute(:metric, metric)
+      set_direction
+    end
+
+    def evaluator
+      {
+        metric: metric,
+        max: direction == "maximize" ? threshold : nil,
+        min: direction == "minimize" ? threshold : nil,
+        direction: direction,
+      }.compact
+    end
+
+    def formatted_frequency
+      {
+        month: "Monthly",
+        week: "Weekly",
+        day: "Daily",
+      }[frequency.to_sym]
+    end
+
+    private
+
+    def metric_class
+      return nil unless metric
+
+      EasyML::Core::ModelEvaluator.get(metric).new
+    end
+
+    def set_direction
+      return unless metric_class.present?
+
+      write_attribute(:direction, metric_class.direction)
+    end
+
+    def validate_at_format
+      return errors.add(:at, "must be a hash") unless at.is_a?(Hash)
+      return if VALID_FREQUENCIES.exclude?(frequency.to_s)
+
+      required_keys = case frequency
+        when "day"
+          ["hour"]
+        when "week"
+          ["hour", "day_of_week"]
+        when "month"
+          ["hour", "day_of_month"]
+        end
+
+      defaults = {
+        "hour" => 0,
+        "day_of_week" => 0, # Sunday
+        "day_of_month" => 1,
+      }
+
+      missing_keys = required_keys - at.keys.map(&:to_s)
+      missing_keys.each do |key|
+        at[key] = defaults[key]
+      end
+
+      return if at.blank?
+
+      allowed_keys = case frequency
+        when "day"
+          ["hour"]
+        when "week"
+          ["hour", "day_of_week"]
+        when "month"
+          ["hour", "day_of_month"]
+        end
+
+      self.at = self.at.select { |k, v| allowed_keys.include?(k.to_s) }.to_h
+
+      if at["hour"].present?
+        errors.add(:at, "hour must be between 0 and 23") unless (0..23).include?(at["hour"].to_i)
+      end
+
+      if at["day_of_week"].present?
+        errors.add(:at, "day_of_week must be between 0 and 6") unless (0..6).include?(at["day_of_week"].to_i)
+      end
+
+      if at["day_of_month"].present?
+        errors.add(:at, "day_of_month must be between 1 and 31") unless (1..31).include?(at["day_of_month"].to_i)
+      end
+    end
+
+    def current_period_start
+      current_time = Time.current
+      case frequency
+      when "hour"
+        current_time.beginning_of_hour
+      when "day"
+        current_time.beginning_of_day
+      when "week"
+        current_time.beginning_of_week
+      when "month"
+        current_time.beginning_of_month
+      end
+    end
+
+    def evaluator_must_be_valid
+      return if evaluator.nil? || evaluator.blank?
+
+      evaluator = self.evaluator.symbolize_keys
+
+      unless evaluator[:metric].present? && (evaluator[:min].present? || evaluator[:max].present?)
+        errors.add(:evaluator, "must specify metric and either min or max value")
+        return
+      end
+
+      errors.add(:evaluator, "min value must be numeric") if evaluator[:min].present? && !evaluator[:min].is_a?(Numeric)
+
+      errors.add(:evaluator, "max value must be numeric") if evaluator[:max].present? && !evaluator[:max].is_a?(Numeric)
+
+      metric = evaluator[:metric].to_sym
+
+      evaluator = EasyML::Core::ModelEvaluator.get(metric)
+      unless evaluator.present?
+        allowed_metrics = EasyML::Core::ModelEvaluator.metrics
+        errors.add(:evaluator, "contains invalid metric. Allowed metrics are #{allowed_metrics}")
+        return
+      end
+
+      return unless evaluator.present?
+      return if evaluator.new.respond_to?(:evaluate)
+
+      errors.add(:evaluator, "evaluator must implement evaluate method")
+    end
+
+    def validate_metrics_allowed
+      return unless metric
+      metric_unknown = EasyML::Core::ModelEvaluator.metrics.exclude?(metric.to_sym)
+      return unless metric_unknown
+
+      errors.add(:metrics,
+                 "don't know how to handle metric #{metric}, use EasyML::Core::ModelEvaluator.register(:name, Evaluator, :regression|:classification)")
+    end
+  end
+end

data/app/models/easy_ml/retraining_run.rb

@@ -0,0 +1,184 @@
+# == Schema Information
+#
+# Table name: easy_ml_retraining_runs
+#
+#  id                  :bigint           not null, primary key
+#  model_id            :bigint
+#  model_history_id    :bigint
+#  model_file_id       :bigint
+#  retraining_job_id   :bigint           not null
+#  tuner_job_id        :bigint
+#  status              :string           default("pending")
+#  metric_value        :float
+#  threshold           :float
+#  trigger             :string           default("manual")
+#  threshold_direction :string
+#  started_at          :datetime
+#  completed_at        :datetime
+#  error_message       :text
+#  metadata            :jsonb
+#  metrics             :jsonb
+#  best_params         :jsonb
+#  wandb_url           :string
+#  snapshot_id         :string
+#  deployable          :boolean
+#  is_deploying        :boolean
+#  deployed            :boolean
+#  deploy_id           :bigint
+#  created_at          :datetime         not null
+#  updated_at          :datetime         not null
+#
+module EasyML
+  class RetrainingRun < ActiveRecord::Base
+    self.table_name = "easy_ml_retraining_runs"
+
+    belongs_to :retraining_job
+    belongs_to :model, class_name: "EasyML::Model"
+    belongs_to :model_file, class_name: "EasyML::ModelFile", optional: true
+    has_many :events, as: :eventable, class_name: "EasyML::Event", dependent: :destroy
+
+    validates :status, presence: true, inclusion: { in: %w[pending running success failed deployed] }
+
+    scope :running, -> { where(status: "running") }
+
+    def deploy(async: true)
+      deploy = EasyML::Deploy.create!(
+        model: model,
+        retraining_run: self,
+      )
+
+      deploy.deploy(async: async)
+    end
+
+    def wrap_training(&block)
+      return false unless pending?
+
+      begin
+        EasyML::Event.create_event(self, "started")
+        update!(status: "running", started_at: Time.current)
+
+        training_model, best_params = yield
+
+        if best_params.present?
+          tuner = EasyML::TunerJob.where(model: training_model)
+            .order(id: :desc)
+            .first
+        end
+
+        results = metric_results(training_model)
+        failed_reasons = training_model.cannot_deploy_reasons - ["Model has not changed"]
+        if results[:deployable] == false
+          status = "success"
+        else
+          status = failed_reasons.any? ? "failed" : "success"
+        end
+
+        if status == "success"
+          training_model.save_model_file
+        end
+
+        update!(
+          results.merge!(
+            status: status,
+            completed_at: failed_reasons.none? ? Time.current : nil,
+            error_message: failed_reasons.any? ? failed_reasons&.first : nil,
+            model: training_model,
+            metrics: training_model.evaluate,
+            best_params: best_params,
+            tuner_job_id: tuner&.id,
+            metadata: tuner&.metadata,
+            wandb_url: tuner&.wandb_url,
+            model_file_id: status == "success" ? training_model.model_file_id : nil,
+          )
+        )
+
+        if failed_reasons.any?
+          EasyML::Event.handle_error(self, failed_reasons.first)
+        else
+          EasyML::Event.create_event(self, status)
+        end
+        params = { last_run_at: Time.current, last_tuning_at: best_params.present? ? Time.current : nil }.compact
+        retraining_job.update!(params)
+
+        reload
+        if deployable? && retraining_job.auto_deploy
+          training_model.save_model_file
+          training_model.reload
+          deploy = EasyML::Deploy.create!(retraining_run: self, model: training_model, model_file: training_model.model_file, trigger: trigger)
+          deploy.deploy
+        end
+        true
+      rescue => e
+        EasyML::Event.handle_error(self, e)
+        update!(
+          status: "failed",
+          completed_at: Time.current,
+          error_message: e.message,
+        )
+        false
+      end
+    end
+
+    def pending?
+      status == "pending"
+    end
+
+    def deployed?
+      status == "deployed"
+    end
+
+    def success?
+      status == "success"
+    end
+
+    def failed?
+      status == "failed"
+    end
+
+    def running?
+      status == "running"
+    end
+
+    def should_tune?
+      retraining_job.tuner_config.present? && retraining_job.should_tune?
+    end
+
+    private
+
+    def metric_results(training_model)
+      return training_model.deployable? unless retraining_job.evaluator.present?
+
+      training_model.dataset.refresh
+      evaluator = retraining_job.evaluator.symbolize_keys
+      x_true, y_true = training_model.dataset.test(split_ys: true)
+      y_pred = training_model.predict(x_true)
+
+      metric = evaluator[:metric].to_sym
+      metrics = EasyML::Core::ModelEvaluator.evaluate(
+        model: training_model,
+        y_pred: y_pred,
+        y_true: y_true,
+        evaluator: evaluator,
+      )
+      metric_value = metrics[metric]
+
+      # Check against min threshold if present
+      if evaluator[:min].present?
+        threshold = evaluator[:min]
+        threshold_direction = "minimize"
+        deployable = metric_value < threshold
+      else
+        threshold = evaluator[:max]
+        threshold_direction = "maximize"
+        deployable = metric_value > threshold
+      end
+
+      {
+        metric_value: metric_value,
+        threshold: threshold,
+        threshold_direction: threshold_direction,
+        deployable: deployable,
+      }
+    end
+  end
+end

data/app/models/easy_ml/settings.rb

@@ -0,0 +1,37 @@
+# == Schema Information
+#
+# Table name: easy_ml_settings
+#
+#  id            :bigint           not null, primary key
+#  configuration :json
+#  created_at    :datetime         not null
+#  updated_at    :datetime         not null
+#
+require_relative "concerns/configurable"
+
+module EasyML
+  class Settings < ActiveRecord::Base
+    self.table_name = "easy_ml_settings"
+    include EasyML::Concerns::Configurable
+
+    add_configuration_attributes :storage,
+      :s3_access_key_id, :s3_secret_access_key,
+      :s3_bucket, :s3_region, :s3_prefix, :timezone,
+      :wandb_api_key
+
+    validates :storage, inclusion: { in: %w[file s3] }, if: -> { storage.present? }
+
+    TIMEZONES = [
+      { value: "America/New_York", label: "Eastern Time" },
+      { value: "America/Chicago", label: "Central Time" },
+      { value: "America/Denver", label: "Mountain Time" },
+      { value: "America/Los_Angeles", label: "Pacific Time" },
+    ]
+
+    def self.constants
+      {
+        TIMEZONES: TIMEZONES,
+      }
+    end
+  end
+end

data/app/models/easy_ml/splitter.rb

@@ -0,0 +1,90 @@
+# == Schema Information
+#
+# Table name: easy_ml_splitters
+#
+#  id            :bigint           not null, primary key
+#  splitter_type :string           not null
+#  configuration :json
+#  dataset_id    :bigint           not null
+#  created_at    :datetime         not null
+#  updated_at    :datetime         not null
+#
+module EasyML
+  class Splitter < ActiveRecord::Base
+    self.table_name = "easy_ml_splitters"
+    include Historiographer::Silent
+    historiographer_mode :snapshot_only
+
+    include EasyML::Concerns::Configurable
+
+    SPLITTER_OPTIONS = {
+      "date" => "EasyML::Splitters::DateSplitter",
+      "random" => "EasyML::Splitters::RandomSplitter",
+      "predefined" => "EasyML::Splitters::PredefinedSplitter",
+    }
+    SPLITTER_TYPES = [
+      {
+        value: "date",
+        label: "Date Splitter",
+        description: "Split dataset based on date ranges for training, validation, and testing",
+      },
+      {
+        value: "random",
+        label: "Random Splitter",
+        description: "Randomly split dataset into training, validation, and testing sets with configurable ratios",
+      },
+      {
+        value: "predefined",
+        label: "Predefined Splitter",
+        description: "Split dataset using predefined file assignments for training, validation, and testing sets",
+      },
+    ].freeze
+
+    belongs_to :dataset, class_name: "EasyML::Dataset"
+    has_many :events, as: :eventable, class_name: "EasyML::Event", dependent: :destroy
+
+    validates :splitter_type, presence: true
+    validates :splitter_type, inclusion: { in: SPLITTER_OPTIONS.keys }
+
+    SPLITTER_NAMES = SPLITTER_OPTIONS.keys.freeze
+    SPLITTER_CONSTANTS = SPLITTER_OPTIONS.values.map(&:constantize)
+    SPLITTER_CONSTANTS.flat_map(&:configuration_attributes).each do |attribute|
+      add_configuration_attributes attribute
+    end
+
+    def self.constants
+      {
+        SPLITTER_TYPES: SPLITTER_TYPES,
+        DEFAULT_CONFIGS: SPLITTER_OPTIONS.reduce({}) do |h, (type, klass)|
+          h.tap do
+            h[type] = klass.constantize.default_config
+          end
+        end,
+      }
+    end
+
+    def split(df, &block)
+      adapter.split(df, &block)
+    end
+
+    def splits
+      adapter.splits
+    end
+
+    private
+
+    def adapter
+      @adapter ||= begin
+          adapter_class = SPLITTER_OPTIONS[splitter_type]
+          raise "Don't know how to use splitter #{splitter_type}!" unless adapter_class.present?
+
+          attrs = adapter_class.constantize.configuration_attributes
+          adapter_class.constantize.new(self).tap do |adapter|
+            attrs.each do |attr|
+              adapter.send("#{attr}=", send(attr))
+            end
+          end
+        end
+    end
+  end
+end

data/app/models/easy_ml/splitters/base_splitter.rb

@@ -0,0 +1,28 @@
+module EasyML
+  module Splitters
+    class BaseSplitter
+      include ActiveModel::Validations
+      include EasyML::Concerns::Configurable
+
+      attr_reader :splitter
+
+      def split(datasource, &block)
+        datasource.in_batches do |df|
+          split_df(df).tap do |splits|
+            yield splits if block_given?
+          end
+        end
+      end
+
+      def split_df(df)
+        df
+      end
+
+      def initialize(splitter)
+        @splitter = splitter
+      end
+
+      delegate :dataset, to: :splitter
+    end
+  end
+end

data/app/models/easy_ml/splitters/date_splitter.rb

@@ -0,0 +1,91 @@
+# == Schema Information
+#
+# Table name: easy_ml_splitters
+#
+#  id            :bigint           not null, primary key
+#  splitter_type :string           not null
+#  configuration :json
+#  dataset_id    :bigint           not null
+#  created_at    :datetime         not null
+#  updated_at    :datetime         not null
+#
+require_relative "base_splitter"
+
+module EasyML
+  module Splitters
+    class DateSplitter < BaseSplitter
+      validates :date_col, presence: true
+      validates :months_test, presence: true, numericality: { greater_than: 0 }
+      validates :months_valid, presence: true, numericality: { greater_than: 0 }
+
+      attr_accessor :today, :date_col, :months_test, :months_valid
+
+      add_configuration_attributes :today, :date_col, :months_test, :months_valid
+
+      def self.default_config
+        {
+          date_col: "",
+          months_test: 2,
+          months_valid: 2,
+        }
+      end
+
+      def split_df(df)
+        raise "Split by date requires argument: date_col" unless date_col.present?
+
+        df = EasyML::Data::DateConverter.maybe_convert_date(df, date_col)
+
+        unless df[date_col].dtype.is_a?(Polars::Datetime)
+          raise "Date splitter cannot split on non-date col #{date_col}, dtype is #{df[date_col].dtype}"
+        end
+
+        validation_date_start, test_date_start = splits
+
+        test_df = Polars.concat(
+          [
+            df.filter(Polars.col(date_col) >= test_date_start),
+            df.filter(Polars.col(date_col).is_null),
+          ]
+        )
+        remaining_df = df.filter(Polars.col(date_col) < test_date_start)
+        valid_df = remaining_df.filter(Polars.col(date_col) >= validation_date_start)
+        train_df = remaining_df.filter(Polars.col(date_col) < validation_date_start)
+
+        [train_df, valid_df, test_df]
+      end
+
+      def months(n)
+        ActiveSupport::Duration.months(n)
+      end
+
+      def splits
+        reference_date = to_datetime(datasource_end) || today
+        test_date_start = reference_date.advance(months: -months_test).beginning_of_day
+        validation_date_start = test_date_start.advance(months: -months_valid).beginning_of_day
+        [validation_date_start, test_date_start]
+      end
+
+      def datasource_end
+        return @datasource_end if @datasource_end
+
+        @datasource_end = dataset.datasource.query(sort: date_col, descending: true, limit: 1,
+                                                   select: date_col)[date_col]&.to_a&.first
+      end
+
+      def to_datetime(field, default: nil)
+        case field
+        when String
+          UTC.parse(field)
+        when NilClass
+          default
+        else
+          field
+        end
+      end
+
+      def today
+        to_datetime(@today, default: UTC.today)
+      end
+    end
+  end
+end