easy_ml 0.2.0.pre.rc71 → 0.2.0.pre.rc75

data/app/jobs/easy_ml/reaper.rb

@@ -0,0 +1,55 @@
+module EasyML
+  class Reaper
+    class << self
+      def list_workers
+        Resque::Worker.all.map do |worker|
+          if worker.working?
+            job = worker.job
+            args = job.dig("payload", "args")&.first
+            {
+              worker: worker,
+              working: true,
+              class: args.dig("job_class"),
+              args: args.dig("arguments"),
+              pid: worker.pid,
+            }
+          else
+            { working: false, class: nil, pid: nil }
+          end
+        end
+      end
+
+      def find_job(worker_class, *args)
+        list_workers.select do |config|
+          config.dig(:class) == worker_class.to_s && config.dig(:args) == args
+        end
+      end
+
+      def kill(worker_class, *args)
+        find_job(worker_class, *args).each do |job|
+          begin
+            # Send TERM signal to the process
+            Process.kill("TERM", job[:pid])
+
+            # Remove the worker from Redis so it doesn't show up as a zombie
+            # in the Resque web interface. This is important because:
+            # 1. It keeps the interface clean
+            # 2. Prevents confusion about running workers
+            # 3. Allows proper worker cleanup in Redis
+            job[:worker].done_working
+            job[:worker].unregister_worker
+          rescue Errno::ESRCH
+            # Process already gone, but still try to clean up Redis
+            begin
+              job[:worker].done_working
+              job[:worker].unregister_worker
+            rescue => e
+              # Redis cleanup failed, worker might already be unregistered
+              puts "Failed to unregister worker: #{e.message}"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/jobs/easy_ml/training_job.rb

@@ -2,7 +2,7 @@ module EasyML
   class TrainingJob < ApplicationJob
     class TrainingTimeoutError < StandardError; end
 
-    INACTIVITY_TIMEOUT = 15 # seconds
+    INACTIVITY_TIMEOUT = 300 # seconds
 
     def perform(model_id)
       @model = EasyML::Model.find_by(id: model_id)

data/app/models/easy_ml/column/imputers/base.rb

@@ -27,6 +27,10 @@ module EasyML
           @preprocessing_step = preprocessing_step.with_indifferent_access
         end
 
+        def expr
+          Polars.col(column.name)
+        end
+
         def applies?
           method_applies? || param_applies?
         end

data/app/models/easy_ml/column/imputers/clip.rb

@@ -10,10 +10,12 @@ module EasyML
           "Clip"
         end
 
+        def expr
+          Polars.col(column.name).clip(min, max).alias(column.name)
+        end
+
         def transform(df)
-          df = df.with_column(
-            Polars.col(column.name).clip(min, max).alias(column.name)
-          )
+          df = df.with_column(expr)
           df
         end
 

data/app/models/easy_ml/column/imputers/imputer.rb

@@ -2,18 +2,23 @@ module EasyML
   class Column
     class Imputers
       class Imputer
-        attr_accessor :dataset, :column, :preprocessing_step
+        attr_accessor :dataset, :column, :preprocessing_step, :allowed_adapters
 
-        def initialize(column, preprocessing_step)
+        def initialize(column, preprocessing_step, allowed_adapters = [])
           @column = column
           @dataset = column.dataset
           @preprocessing_step = preprocessing_step.with_indifferent_access
+          @allowed_adapters = allowed_adapters.map(&:to_sym)
         end
 
         def inspect
           "#<#{self.class.name} adapters=#{adapters.map(&:inspect).join(", ")}>"
         end
 
+        def exprs
+          adapters.map(&:expr)
+        end
+
         def ordered_adapters
           [
             Clip,
@@ -29,19 +34,12 @@ module EasyML
           ]
         end
 
-        def adapters
-          @adapters ||= ordered_adapters.map { |klass| klass.new(column, preprocessing_step) }.select(&:applies?)
+        def allowed?(adapter)
+          allowed_adapters.empty? || allowed_adapters.include?(adapter.class.name.split("::").last.underscore.to_sym)
         end
 
-        def imputers
-          return nil if column.preprocessing_steps.blank?
-
-          @imputers ||= column.preprocessing_steps.keys.reduce({}) do |hash, key|
-            hash[key.to_sym] = Imputer.new(
-              column: column,
-              preprocessing_step: column.preprocessing_steps[key],
-            )
-          end
+        def adapters
+          @adapters ||= ordered_adapters.map { |klass| klass.new(column, preprocessing_step) }.select { |adapter| allowed?(adapter) && adapter.applies? }
         end
 
         def description

data/app/models/easy_ml/column/imputers/mean.rb

@@ -8,13 +8,17 @@ module EasyML
           "Mean imputation"
         end
 
+        def expr
+          return super unless mean.present?
+
+          Polars.col(column.name).fill_null(mean).alias(column.name)
+        end
+
         def transform(df)
           return df unless mean.present?
 
           mean = statistics(:mean)
-          df = df.with_column(
-            Polars.col(column.name).fill_null(mean).alias(column.name)
-          )
+          df = df.with_column(expr)
           df
         end
 

data/app/models/easy_ml/column/imputers/null_imputer.rb

@@ -6,6 +6,9 @@ module EasyML
           false
         end
 
+        def exprs
+        end
+
         def method_missing(_name, df)
           df
         end

data/app/models/easy_ml/column/imputers/ordinal_encoder.rb

@@ -50,7 +50,11 @@ module EasyML
         end
 
         def cast_encoder(encoder)
-          encoder.transform_keys { |k| column.cast(k) }
+          begin
+            encoder.transform_keys { |k| column.cast(k) }
+          rescue => e
+            binding.pry
+          end
         end
 
         def cast_decoder(decoder)

data/app/models/easy_ml/column/imputers.rb

@@ -74,9 +74,10 @@ module EasyML
         @supported_methods ||= []
       end
 
-      def initialize(column)
+      def initialize(column, imputers: [])
         @column = column
         @dataset = column.dataset
+        @_imputers = imputers
       end
 
       class << self
@@ -97,6 +98,7 @@ module EasyML
             hash[key.to_sym] = Imputer.new(
               column,
               column.preprocessing_steps[key],
+              @_imputers
             )
           end
         end

data/app/models/easy_ml/column/lineage/base.rb

@@ -9,11 +9,15 @@ module EasyML
           @dataset = column.dataset
         end
 
+        def expr
+          Polars.col(column.name)
+        end
+
         def as_json
           {
             key: key,
             description: description,
-            timestamp: timestamp,
+            occurred_at: occurred_at,
           }.with_indifferent_access
         end
       end

data/app/models/easy_ml/column/lineage/computed_by_feature.rb

@@ -10,7 +10,7 @@ module EasyML
           "Computed by #{column.computed_by}"
         end
 
-        def timestamp
+        def occurred_at
           column.feature.fit_at || column.feature.applied_at
         end
 

data/app/models/easy_ml/column/lineage/preprocessed.rb

@@ -10,7 +10,7 @@ module EasyML
           "Preprocessed using #{column.imputers.preprocessing_descriptions.join(", ")}"
         end
 
-        def timestamp
+        def occurred_at
           column.dataset.refreshed_at
         end
 

data/app/models/easy_ml/column/lineage/raw_dataset.rb

@@ -10,7 +10,7 @@ module EasyML
           "Present in raw dataset"
         end
 
-        def timestamp
+        def occurred_at
           column.dataset.datasource.refreshed_at
         end
 

data/app/models/easy_ml/column/selector.rb

@@ -1,6 +1,8 @@
 module EasyML
   class Column
     class Selector
+      include EasyML::Timing
+
       attr_accessor :selected, :dataset, :column, :transform
 
       def initialize(column, selected = nil, &block)
@@ -28,6 +30,8 @@ module EasyML
         end
       end
 
+      measure_method_timing :clipped
+
       def processed
         Selector.new(column, :processed)
       end

data/app/models/easy_ml/column.rb

@@ -2,29 +2,29 @@
 #
 # Table name: easy_ml_columns
 #
-#  id                       :bigint           not null, primary key
-#  dataset_id               :bigint           not null
-#  name                     :string           not null
-#  description              :string
-#  datatype                 :string
-#  polars_datatype          :string
-#  is_target                :boolean          default(FALSE)
-#  hidden                   :boolean          default(FALSE)
-#  drop_if_null             :boolean          default(FALSE)
-#  preprocessing_steps      :json
-#  sample_values            :json
-#  statistics               :json
-#  created_at               :datetime         not null
-#  updated_at               :datetime         not null
-#  is_date_column           :boolean          default(FALSE)
-#  computed_by              :string
-#  is_computed              :boolean          default(FALSE)
-#  feature_id               :bigint
-#  learned_at               :datetime
-#  is_learning              :boolean          default(FALSE)
-#  last_datasource_sha      :string
-#  last_feature_sha         :string
-#  configuration_changed_at :datetime
+#  id                  :bigint           not null, primary key
+#  dataset_id          :bigint           not null
+#  name                :string           not null
+#  description         :string
+#  datatype            :string
+#  polars_datatype     :string
+#  is_target           :boolean          default(FALSE)
+#  hidden              :boolean          default(FALSE)
+#  drop_if_null        :boolean          default(FALSE)
+#  preprocessing_steps :jsonb
+#  sample_values       :json
+#  statistics          :json
+#  created_at          :datetime         not null
+#  updated_at          :datetime         not null
+#  is_date_column      :boolean          default(FALSE)
+#  computed_by         :string
+#  is_computed         :boolean          default(FALSE)
+#  feature_id          :bigint
+#  learned_at          :datetime
+#  is_learning         :boolean          default(FALSE)
+#  last_datasource_sha :string
+#  last_feature_sha    :string
+#  in_raw_dataset      :boolean
 #
 module EasyML
   class Column < ActiveRecord::Base
@@ -32,8 +32,11 @@ module EasyML
     include Historiographer::Silent
     historiographer_mode :snapshot_only
 
+    include EasyML::Timing
+
     belongs_to :dataset, class_name: "EasyML::Dataset"
     belongs_to :feature, class_name: "EasyML::Feature", optional: true
+    has_many :lineages, class_name: "EasyML::Lineage"
 
     validates :name, presence: true
     validates :name, uniqueness: { scope: :dataset_id }
@@ -43,7 +46,7 @@ module EasyML
     before_save :set_defaults
     before_save :set_feature_lineage
     before_save :set_polars_datatype
-    after_find :ensure_feature_exists
+    # after_find :ensure_feature_exists
 
     # Scopes
     scope :visible, -> { where(hidden: false) }
@@ -60,6 +63,7 @@ module EasyML
     scope :api_inputs, -> { where(is_computed: false, hidden: false, is_target: false) }
     scope :computed, -> { where(is_computed: true) }
     scope :raw, -> { where(is_computed: false) }
+    scope :has_clip, -> { where("preprocessing_steps->'training'->>'params' IS NOT NULL AND preprocessing_steps->'training'->'params' @> jsonb_build_object('clip', jsonb_build_object())") }
     scope :needs_learn, -> {
             datasource_changed
               .or(feature_applied)
@@ -142,26 +146,10 @@ module EasyML
       data.blank?
     end
 
-    def learn(type: :all)
-      return if (!in_raw_dataset? && type != :processed)
-
-      if !in_raw_dataset? && read_attribute(:datatype).nil?
-        assign_attributes(datatype: processed.data.to_series.dtype)
-      end
-      set_sample_values
-      new_stats = learner.learn(type: type).symbolize_keys
-
-      if !in_raw_dataset?
-        new_stats[:raw] = new_stats[:processed]
-      end
+    def merge_statistics(new_stats)
+      return unless new_stats.present?
 
-      assign_attributes(statistics: (read_attribute(:statistics) || {}).symbolize_keys.merge!(new_stats))
-      assign_attributes(
-        learned_at: UTC.now,
-        last_datasource_sha: dataset.last_datasource_sha,
-        last_feature_sha: feature&.sha,
-        is_learning: type == :raw,
-      )
+      assign_attributes(statistics: (statistics || {}).symbolize_keys.deep_merge!(new_stats))
     end
 
     def set_configuration_changed_at
@@ -174,7 +162,7 @@ module EasyML
       use_processed = !one_hot? && processed.data(limit: 1).present? && in_raw_dataset?
 
       base = use_processed ? processed : raw
-      sample_values = base.data(limit: 5, unique: true)
+      sample_values = base.data(limit: 5, unique: true, select: [name])
       if sample_values.columns.include?(name)
         sample_values = sample_values[name].to_a.uniq[0...5]
         assign_attributes(sample_values: sample_values)
@@ -188,8 +176,8 @@ module EasyML
       df
     end
 
-    def imputers
-      @imputers ||= Column::Imputers.new(self)
+    def imputers(imputers = [])
+      @imputers ||= Column::Imputers.new(self, imputers: imputers)
     end
 
     def decode_labels(df)
@@ -202,29 +190,29 @@ module EasyML
 
     def datatype=(dtype)
       if dtype.is_a?(Polars::DataType)
-        dtype = EasyML::Data::PolarsColumn.polars_to_sym(dtype)
+        dtype = polars_to_sym(dtype)
       end
       write_attribute(:datatype, dtype)
       set_polars_datatype
     end
 
+    def polars_to_sym(dtype)
+      EasyML::Data::PolarsColumn.polars_to_sym(dtype)
+    end
+
     def datatype
-      read_attribute(:datatype) || write_attribute(:datatype, assumed_datatype)
+      read_attribute(:datatype) || write_attribute(:datatype, polars_to_sym(assumed_datatype))
     end
 
     def raw_dtype
-      return @raw_dtype if @raw_dtype
-      set_feature_lineage
+      dtype = dataset.raw_schema[name]
+      return nil if dtype.nil?
 
-      if in_raw_dataset?
-        @raw_dtype = raw&.data&.to_series.try(:dtype)
-      elsif already_computed?
-        @raw_dtype = processed&.data&.to_series&.dtype
-      end
+      polars_to_sym(dtype)
     end
 
     def set_polars_datatype
-      raw_type = raw_dtype
+      raw_type = datatype
       user_type = get_polars_type(datatype)
 
       if raw_type == user_type
@@ -267,8 +255,11 @@ module EasyML
       return @assumed_datatype if @assumed_datatype
 
       if in_raw_dataset?
-        series = (raw.data || datasource_raw).to_series
-        @assumed_datatype = EasyML::Data::PolarsColumn.determine_type(series)
+        @assumed_datatype = dataset.raw_schema[name]
+        # series = (raw.data || datasource_raw).to_series
+        # @assumed_datatype = EasyML::Data::PolarsColumn.determine_type(series)
+      elsif dataset.processed_schema.present?
+        @assumed_datatype = dataset.processed_schema[name]
       elsif already_computed?
         return nil if processed.data.nil?
 
@@ -277,9 +268,16 @@ module EasyML
     end
 
     def in_raw_dataset?
+      value = read_attribute(:in_raw_dataset)
+      return value unless value.nil?
+
+      write_attribute(:in_raw_dataset, check_in_raw_dataset?)
+    end
+
+    def check_in_raw_dataset?
       return false if dataset&.raw&.data.nil?
 
-      dataset.raw.data(all_columns: true)&.columns&.include?(name) || false
+      dataset.raw.data(all_columns: true, lazy: true).schema.key?(name) || false
     end
 
     def computing_feature
@@ -398,10 +396,6 @@ module EasyML
       is_date_column
     end
 
-    def lineage
-      @lineage ||= EasyML::Column::Lineage.new(self).lineage
-    end
-
     def required?
       !is_computed && (preprocessing_steps.nil? || preprocessing_steps == {}) && !hidden && !is_target
     end
@@ -420,6 +414,28 @@ module EasyML
       }.compact
     end
 
+    UNCONFIGURABLE_COLUMNS = %w(
+      id
+      feature_id
+      dataset_id
+      last_datasource_sha
+      last_feature_sha
+      learned_at
+      is_learning
+      configuration_changed_at
+      statistics
+      created_at
+      updated_at
+    )
+
+    def to_config
+      EasyML::Export::Column.to_config(self)
+    end
+
+    def self.from_config(config, dataset, action: :create)
+      EasyML::Import::Column.from_config(config, dataset, action: action)
+    end
+
     def cast(value)
       return value if value.nil?
 

data/app/models/easy_ml/column_history.rb

@@ -2,34 +2,34 @@
 #
 # Table name: easy_ml_column_histories
 #
-#  id                       :bigint           not null, primary key
-#  column_id                :integer          not null
-#  dataset_id               :integer          not null
-#  name                     :string           not null
-#  description              :string
-#  datatype                 :string
-#  polars_datatype          :string
-#  is_target                :boolean          default(FALSE)
-#  hidden                   :boolean          default(FALSE)
-#  drop_if_null             :boolean          default(FALSE)
-#  preprocessing_steps      :json
-#  sample_values            :json
-#  statistics               :json
-#  created_at               :datetime         not null
-#  updated_at               :datetime         not null
-#  history_started_at       :datetime         not null
-#  history_ended_at         :datetime
-#  history_user_id          :integer
-#  snapshot_id              :string
-#  is_date_column           :boolean          default(FALSE)
-#  computed_by              :string
-#  is_computed              :boolean          default(FALSE)
-#  feature_id               :bigint
-#  learned_at               :datetime
-#  is_learning              :boolean          default(FALSE)
-#  last_datasource_sha      :string
-#  last_feature_sha         :string
-#  configuration_changed_at :datetime
+#  id                  :bigint           not null, primary key
+#  column_id           :integer          not null
+#  dataset_id          :integer          not null
+#  name                :string           not null
+#  description         :string
+#  datatype            :string
+#  polars_datatype     :string
+#  is_target           :boolean          default(FALSE)
+#  hidden              :boolean          default(FALSE)
+#  drop_if_null        :boolean          default(FALSE)
+#  preprocessing_steps :jsonb
+#  sample_values       :json
+#  statistics          :json
+#  created_at          :datetime         not null
+#  updated_at          :datetime         not null
+#  history_started_at  :datetime         not null
+#  history_ended_at    :datetime
+#  history_user_id     :integer
+#  snapshot_id         :string
+#  is_date_column      :boolean          default(FALSE)
+#  computed_by         :string
+#  is_computed         :boolean          default(FALSE)
+#  feature_id          :bigint
+#  learned_at          :datetime
+#  is_learning         :boolean          default(FALSE)
+#  last_datasource_sha :string
+#  last_feature_sha    :string
+#  in_raw_dataset      :boolean
 #
 module EasyML
   class ColumnHistory < ActiveRecord::Base

data/app/models/easy_ml/column_list/imputer.rb

@@ -0,0 +1,23 @@
+module EasyML
+  module ColumnList
+    class Imputer
+      attr_accessor :dataset, :df, :inference, :columns
+
+      def initialize(dataset, df, columns: nil, imputers: [], inference: false)
+        @dataset = dataset
+        @df = df
+        @columns = (columns.nil? || columns.empty?) ? dataset.columns : columns
+        @inference = inference
+        @_imputers = imputers
+      end
+
+      def imputers
+        @imputers ||= columns.map { |column| inference ? column.imputers(@_imputers).inference : column.imputers(@_imputers).training }
+      end
+
+      def exprs
+        imputers.flat_map(&:exprs).compact
+      end
+    end
+  end
+end