easy_ml 0.2.0.pre.rc77 → 0.2.0.pre.rc81

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ee980703e3a768458e43d54a878bfa712d4f026967f3ccd8fa5bb2d1df50304c
-  data.tar.gz: eb5eb31b580e9112886527f416d4f360ffe9b0ee73d9e2e7dd70d9a48528ea09
+  metadata.gz: 169873e9ea5e1b00f7a4e499a2aeffc377615757cdd4b5fe6d70c8c454b9d426
+  data.tar.gz: fc1d4509606f011bd3adbdf367e767a3e9dfc4fdbb6b5cd91bb413f72da364b2
 SHA512:
-  metadata.gz: ddc8a0005b22caf186c13790e9209d7b843181d62be8e70710e33bd6e244e3f31e6cc02efc4effa82d9f38674ff8ba2d8abdc1e43db8ba339d712f0e12d10ec4
-  data.tar.gz: 52193a2c0da5c0aca86bb627afff7efd9cbdfcf80ba2db8ae4e9baec061b716244e14c49f30451f52a221700b2b1a160a71935cb76cd77732a4e8fdcf25bd3a1
+  metadata.gz: f4ea106a66d3185f612e481b607cef21f5453a00ef6eb12558e53f43aa1d68b8f20d1e950c1840d97180ba6271db9b9b42c8a2d480c22ca8c1e33d1b768590d2
+  data.tar.gz: 24885cdecd46d612be8b8ce7d4bd9889bdf60420bc82c01993cfe0168e454ebdaaa70899f5cf9cc879ff89a895fd00b1761d01c945dafffc784f7bc1e4a2fb8e

data/app/controllers/easy_ml/datasets_controller.rb

@@ -23,7 +23,7 @@
 module EasyML
   class DatasetsController < ApplicationController
     def index
-      datasets = Dataset.all.order(id: :desc)
+      datasets = Dataset.all.includes(:columns, :datasource).order(id: :desc)
 
       render inertia: "pages/DatasetsPage", props: {
         datasets: datasets.map { |dataset| dataset_to_json_small(dataset) },
@@ -80,7 +80,7 @@ module EasyML
       if dataset_params[:features_attributes].present?
         # Clean up any feature IDs that don't exist anymore
         feature_ids = dataset_params[:features_attributes].map { |attrs| attrs[:id] }.compact
-        existing_feature_ids = Feature.where(id: feature_ids).pluck(:id)
+        existing_feature_ids = dataset.features.where(id: feature_ids).pluck(:id)
 
         params[:dataset][:features_attributes].each do |attrs|
           if attrs[:id].present? && !existing_feature_ids.include?(attrs[:id].to_i)
@@ -93,7 +93,7 @@ module EasyML
           attrs[:feature_class] if attrs[:id].blank?
         }.compact
 
-        existing_features = Feature.where(feature_class: feature_classes)
+        existing_features = dataset.features.where(feature_class: feature_classes)
 
         # Update params with existing feature IDs
         existing_features.each do |feature|

data/app/controllers/easy_ml/models_controller.rb

@@ -30,7 +30,7 @@ module EasyML
     def new
       render inertia: "pages/NewModelPage", props: {
         datasets: EasyML::Dataset.all.map do |dataset|
-          dataset.slice(:id, :name, :num_rows)
+          dataset_to_json(dataset)
         end,
         constants: EasyML::Model.constants,
       }
@@ -41,7 +41,7 @@ module EasyML
       render inertia: "pages/EditModelPage", props: {
         model: model_to_json(model),
         datasets: EasyML::Dataset.all.map do |dataset|
-          dataset.slice(:id, :name, :num_rows)
+          dataset_to_json_small(dataset)
         end,
         constants: EasyML::Model.constants,
       }
@@ -167,7 +167,7 @@ module EasyML
     private
 
     def includes_list
-      [:retraining_runs, :retraining_job, dataset: [:columns, :features, :splitter]]
+      [:retraining_runs, :retraining_job, dataset: [:features, :splitter, columns: [:lineages]]]
     end
 
     def model_params
@@ -177,6 +177,7 @@ module EasyML
         :dataset_id,
         :task,
         :objective,
+        :weights_column,
         metrics: [],
         retraining_job_attributes: [
           :id,

data/app/frontend/components/ModelForm.tsx

@@ -16,6 +16,7 @@ interface ModelFormProps {
     task: string;
     objective?: string;
     metrics?: string[];
+    weights_column?: string;
     retraining_job?: {
       frequency: string;
       at: {
@@ -75,6 +76,7 @@ export function ModelForm({ initialData, datasets, constants, isEditing, errors:
       task: initialData?.task || 'classification',
       objective: initialData?.objective || 'binary:logistic',
       metrics: initialData?.metrics || ['accuracy_score'],
+      weights_column: initialData?.weights_column || '',
       retraining_job_attributes: initialData?.retraining_job ? {
         id: initialData.retraining_job.id,
         frequency: initialData.retraining_job.frequency,
@@ -165,6 +167,7 @@ export function ModelForm({ initialData, datasets, constants, isEditing, errors:
   };
 
   const selectedDataset = datasets.find(d => d.id === data.model.dataset_id);
+  const columns = selectedDataset?.columns || [];
 
   const filteredTunerJobConstants = constants.tuner_job_constants[data.model.model_type] || {};
 
@@ -246,6 +249,19 @@ export function ModelForm({ initialData, datasets, constants, isEditing, errors:
             <ErrorDisplay error={errors.dataset_id} />
           </div>
 
+          <div>
+            <label className="block text-sm font-medium text-gray-700 mb-1">
+              Weights Column (Optional)
+            </label>
+            <SearchableSelect
+              value={data.model.weights_column}
+              options={columns.map(col => ({ value: col.name, label: col.name }))}
+              onChange={(value) => setData('model.weights_column', value)}
+              isClearable={true}
+            />
+            <ErrorDisplay error={errors.weights_column} />
+          </div>
+
           <div>
             <label className="block text-sm font-medium text-gray-700 mb-1">
               Task

data/app/frontend/components/ScheduleModal.tsx

@@ -587,8 +587,6 @@ export function ScheduleModal({ isOpen, onClose, onSave, initialData, metrics, t
                           value={formData.retraining_job_attributes.threshold}
                           onChange={(e) => handleEvaluatorChange('threshold', parseFloat(e.target.value))}
                           step={0.01}
-                          min={0}
-                          max={1}
                           className="block w-full rounded-md border-gray-300 shadow-sm focus:border-blue-500 focus:ring-blue-500 py-2 px-4 shadow-sm border-gray-300 border"
                         />
                       </div>

data/app/frontend/components/dataset/PreprocessingConfig.tsx

@@ -250,16 +250,17 @@ export function PreprocessingConfig({
     setIsEditingDescription(true);
   };
 
-  let nullCount = (column.statistics?.processed.null_count || column.statistics?.raw?.null_count) || 0;
-  const nullPercentage = nullCount && column.statistics?.raw.num_rows
-    ? ((nullCount / column.statistics.raw.num_rows) * 100)
+  let nullCount = (column.statistics?.processed?.null_count || column.statistics?.raw?.null_count) || 0;
+  let numRows = (column.statistics?.processed?.num_rows) || (column.statistics?.raw?.num_rows) || 0;
+  const nullPercentage = nullCount && numRows
+    ? ((nullCount / numRows) * 100)
     : 0;
 
-  const nullPercentageProcessed = column.statistics?.processed?.null_count && column.statistics?.raw.num_rows
-    ? ((column.statistics.processed.null_count / column.statistics.raw.num_rows) * 100)
+  const nullPercentageProcessed = column.statistics?.processed?.null_count && column.statistics?.processed?.num_rows
+    ? ((column.statistics.processed.null_count / column.statistics.processed.num_rows) * 100)
     : 0;
 
-  const totalRows = column.statistics?.raw.num_rows ?? 0;
+  const totalRows = numRows;
 
   const renderStrategySpecificInfo = (type: 'training' | 'inference') => {
     const strategy = type === 'training' ? training : inference;

data/app/jobs/easy_ml/application_job.rb

@@ -1,5 +1,6 @@
 module EasyML
   class ApplicationJob < ActiveJob::Base
+    @queue = :easy_ml
     queue_as :easy_ml
 
     def create_event(model, status, error = nil)

data/app/jobs/easy_ml/batch_job.rb

@@ -39,15 +39,15 @@ module EasyML
         rest.map do |batch|
           Resque.redis.rpush("batch:#{parent_id}:remaining", batch.to_json)
         end
-
+        track_batch(parent_id)
         handle_batch(parent_id, batch)
       end
 
       def handle_batch(parent_id, batch)
         if batch.size > 1
-          enqueue_batch(batch)
+          enqueue_batch(batch, parent_id)
         else
-          run_one_batch(parent_id, batch.first)
+          new.perform(parent_id, batch.first)
           after_batch_hook(parent_id, batch)
         end
       end
@@ -60,7 +60,21 @@ module EasyML
       end
 
       def next_batch?(parent_id)
-        batches_remaining(parent_id) > 0
+        (batches_remaining(parent_id) > 0)
+      end
+
+      def list_batches
+        Resque.redis.hkeys("batches:tracking")
+      end
+
+      def track_batch(parent_id)
+        Resque.redis.hset("batches:tracking", parent_id, 1)
+      end
+
+      def cleanup_all
+        list_batches.each do |batch_id|
+          cleanup_batch(batch_id)
+        end
       end
 
       def batches_remaining(parent_id)
@@ -69,12 +83,39 @@ module EasyML
 
       def cleanup_batch(parent_id)
         Resque.redis.del("batch:#{parent_id}:remaining")
+        Resque.redis.hdel("batches:tracking", parent_id)
       end
 
-      private
+      def batch_args
+        list_batches.map do |batch_id|
+          fetch_batch_arguments(batch_id)
+        end
+      end
+
+      def select_batches(&block)
+        list_batches.select do |batch_id|
+          yield fetch_batch_arguments(batch_id)
+        end
+      end
+
+      def poll
+        while true
+          sleep 2
+          EasyML::BatchJob.list_batches.map do |batch|
+            puts "Batch #{batch} | Remaining : #{EasyML::BatchJob.batches_remaining(batch)}"
+          end
+        end
+      end
 
       def get_parent_batch_id(args_list)
-        args_list.dup.flatten.first.dig(:parent_batch_id)
+        args_list.dup.flatten.detect { |arg| arg.dig(:parent_batch_id) }.dig(:parent_batch_id)
+      end
+
+      private
+
+      def get_args_list(batch_id)
+        redis_key = "#{batch(batch_id)}:original_args"
+        redis.get(redis_key)
       end
 
       # Store batch arguments in Redis

data/app/jobs/easy_ml/compute_feature_job.rb

@@ -14,31 +14,31 @@ module EasyML
       #
       # https://github.com/drfeelngood/resque-batched-job/blob/master/lib/resque/plugins/batched_job.rb#L86
       batch_args = batch_args.dup
-      run_one_batch(batch_id, batch_args)
+      EasyML::ComputeFeatureJob.new.perform(batch_id, batch_args)
     end
 
-    def self.run_one_batch(batch_id, batch_args)
+    def perform(batch_id, batch_args = {})
       EasyML::Feature.fit_one_batch(batch_id, batch_args)
     end
 
     def self.after_batch_hook(batch_id, *args)
-      batch_args = fetch_batch_arguments(batch_id).flatten.map(&:symbolize_keys)
-      feature_ids = batch_args.pluck(:feature_id).uniq
-      parent_id = batch_args.pluck(:parent_batch_id).first
+      args = args.flatten.first.with_indifferent_access
+      feature_id = args.dig(:feature_id)
 
-      feature = EasyML::Feature.find_by(id: feature_ids.first)
+      feature = EasyML::Feature.find_by(id: feature_id)
 
       if feature.failed?
         dataset.features.where(workflow_status: :analyzing).update_all(workflow_status: :ready)
-        return BatchJob.cleanup_batch(parent_id)
+        return BatchJob.cleanup_batch(batch_id)
       end
 
       feature.after_fit
 
-      if BatchJob.next_batch?(parent_id)
-        BatchJob.enqueue_next_batch(self, parent_id)
+      if BatchJob.next_batch?(batch_id)
+        BatchJob.enqueue_next_batch(self, batch_id)
       else
-        dataset = EasyML::Feature.find_by(id: feature_ids.first).dataset
+        cleanup_batch(batch_id)
+        dataset = feature.dataset
         dataset.after_fit_features
       end
     end

data/app/jobs/easy_ml/reaper.rb

@@ -9,8 +9,8 @@ module EasyML
             {
               worker: worker,
               working: true,
-              class: args.dig("job_class"),
-              args: args.dig("arguments"),
+              class: args.is_a?(Hash) ? args.dig("job_class") : nil,
+              args: args.is_a?(Hash) ? args.dig("arguments") : nil,
               pid: worker.pid,
             }
           else
@@ -19,17 +19,23 @@ module EasyML
         end
       end
 
-      def find_job(worker_class, *args)
+      def find_job(worker_class, *args, &block)
         list_workers.select do |config|
-          config.dig(:class) == worker_class.to_s && config.dig(:args) == args
+          selected = config.dig(:class) == worker_class.to_s
+          if block_given?
+            selected &&= yield(config)
+          else
+            selected &= config.dig(:args) == args
+          end
+          selected
         end
       end
 
-      def kill(worker_class, *args)
-        find_job(worker_class, *args).each do |job|
+      def kill(worker_class, *args, &block)
+        find_job(worker_class, *args, &block).each do |job|
           begin
-            # Send TERM signal to the process
-            Process.kill("TERM", job[:pid])
+            # Send HUP signal to the process
+            Process.kill("USR1", job[:pid])
 
             # Remove the worker from Redis so it doesn't show up as a zombie
             # in the Resque web interface. This is important because:
@@ -37,12 +43,10 @@ module EasyML
             # 2. Prevents confusion about running workers
             # 3. Allows proper worker cleanup in Redis
             job[:worker].done_working
-            job[:worker].unregister_worker
           rescue Errno::ESRCH
             # Process already gone, but still try to clean up Redis
             begin
               job[:worker].done_working
-              job[:worker].unregister_worker
             rescue => e
               # Redis cleanup failed, worker might already be unregistered
               puts "Failed to unregister worker: #{e.message}"

data/app/jobs/easy_ml/refresh_dataset_job.rb

@@ -1,5 +1,7 @@
 module EasyML
   class RefreshDatasetJob < ApplicationJob
+    @queue = :easy_ml
+
     def perform(id)
       begin
         dataset = EasyML::Dataset.find(id)

data/app/jobs/easy_ml/sync_datasource_job.rb

@@ -8,6 +8,7 @@ module EasyML
 
       begin
         datasource.refresh
+        datasource.after_sync
       rescue StandardError => e
         datasource.update!(is_syncing: false)
         handle_error(datasource, e)

data/app/models/concerns/easy_ml/dataframe_serialization.rb

@@ -8,23 +8,7 @@ module EasyML
     end
 
     def deserialize_dataframe(df_data)
-      return unless df_data.present? && df_data.key?("columns")
-
-      columns = df_data["columns"].map do |col|
-        dtype = case col["datatype"]
-          when Hash
-            if col["datatype"]["Datetime"]
-              Polars::Datetime.new(col["datatype"]["Datetime"][0].downcase.to_sym).class
-            else
-              Polars::Utf8
-            end
-          else
-            Polars.const_get(col["datatype"])
-          end
-        Polars::Series.new(col["name"], col["values"], dtype: dtype)
-      end
-
-      Polars::DataFrame.new(columns)
+      Polars::DataFrame.new(df_data)
     end
   end
 end

data/app/models/easy_ml/column/imputers/base.rb

@@ -63,7 +63,7 @@ module EasyML
           if column.is_computed
             column.statistics.dig(:processed, *args)
           else
-            column.statistics.dig(:clipped, *args) || column.statistics.dig(:raw, *args)
+            column.statistics.dig(:raw, *args)
           end
         end
 

data/app/models/easy_ml/column/imputers/imputer.rb

@@ -54,6 +54,8 @@ module EasyML
           return df unless anything?
 
           adapters.reduce(df) do |df, adapter|
+            next df if df.columns.exclude?(column.name)
+
             adapter.transform(df)
           end
         end

data/app/models/easy_ml/column/imputers/today.rb

@@ -10,7 +10,7 @@ module EasyML
 
         def transform(df)
           df = df.with_column(
-            Polars.col(column.name).fill_null(Polars.lit(UTC.today.beginning_of_day)).alias(column.name)
+            Polars.col(column.name).fill_null(Polars.lit(EasyML::Support::UTC.today.beginning_of_day)).alias(column.name)
           )
           df
         end

data/app/models/easy_ml/column/selector.rb

@@ -24,14 +24,6 @@ module EasyML
         end
       end
 
-      def clipped
-        Selector.new(column, :raw) do |df|
-          column.imputers.training.clip(df)
-        end
-      end
-
-      measure_method_timing :clipped
-
       def processed
         Selector.new(column, :processed)
       end

data/app/models/easy_ml/column.rb

@@ -140,7 +140,7 @@ module EasyML
       end
     end
 
-    delegate :raw, :processed, :data, :train, :test, :valid, :clipped, to: :data_selector
+    delegate :raw, :processed, :data, :train, :test, :valid, to: :data_selector
 
     def empty?
       data.blank?

data/app/models/easy_ml/column_list.rb

@@ -28,12 +28,11 @@ module EasyML
       if computed
         cols = column_list.computed
       else
-        cols = column_list.raw
+        cols = column_list
       end
 
       by_name = cols.index_by(&:name)
-      df.columns.each do |col|
-        column = by_name[col]
+      cols.each do |column|
         df = column.transform(df, inference: inference, computed: computed) if column
       end
 

data/app/models/easy_ml/dataset/learner/base.rb

@@ -15,8 +15,8 @@ module EasyML
             (column.one_hot? && type.to_sym == :processed)
         end
 
-        TYPES_ALL = %i(raw clipped processed)
-        TYPES_RAW = %i(raw clipped)
+        TYPES_ALL = %i(raw processed)
+        TYPES_RAW = %i(raw)
         TYPES_PROCESSED = %i(processed)
 
         def types(type = :all)

data/app/models/easy_ml/dataset/learner/eager.rb

@@ -19,7 +19,9 @@ module EasyML
         end
 
         def fetch_df(split, type)
-          @dataset.send(type).send(split, all_columns: true)
+          dataset.columns.apply_clip(
+            @dataset.send(type).send(split, all_columns: true)
+          )
         end
 
         def execute_queries(split, type)

data/app/models/easy_ml/dataset/learner/lazy.rb

@@ -21,7 +21,10 @@ module EasyML
 
         def run_queries(split, type)
           queries = build_queries(split, type)
-          @dataset.send(type).send(split, all_columns: true, lazy: true).select(queries).collect
+
+          dataset.columns.apply_clip(
+            @dataset.send(type).send(split, all_columns: true, lazy: true)
+          ).select(queries).collect
         end
 
         def get_column_statistics(query_results)