easy_ml 0.1.3 → 0.2.0.pre.rc1

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_datasets.rb.tt

@@ -0,0 +1,31 @@
+class CreateEasyMLDatasets < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    create_table :easy_ml_datasets do |t|
+      t.string :name, null: false
+      t.string :description
+      t.string :dataset_type
+      t.string :status
+      t.string :version
+      t.bigint :datasource_id
+      t.string :root_dir
+      t.json :configuration
+      t.bigint :num_rows
+      t.string :workflow_status
+      t.json :statistics
+      t.json :preprocessor_statistics
+      t.json :schema
+      t.datetime :refreshed_at
+
+      t.timestamps
+
+      t.index :created_at
+      t.index :refreshed_at
+      t.index :name
+      t.index :status
+      t.index [:name, :status]
+      t.index :datasource_id
+      t.index :dataset_type
+      t.index :workflow_status
+    end
+  end
+end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_datasource_histories.rb.tt

@@ -0,0 +1,9 @@
+require "historiographer/postgres_migration"
+
+class CreateEasyMLDatasourceHistories < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    create_table :easy_ml_datasource_histories do |t|
+      t.histories(foreign_key: :datasource_id)
+    end
+  end
+end 

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_datasources.rb.tt

@@ -0,0 +1,16 @@
+class CreateEasyMLDatasources < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    create_table :easy_ml_datasources do |t|
+      t.string :name, null: false
+      t.string :datasource_type
+      t.string :root_dir
+      t.json :configuration
+      t.datetime :refreshed_at
+
+      t.timestamps
+      t.index :created_at
+      t.index :datasource_type
+      t.index :refreshed_at
+    end
+  end
+end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_deploys.rb.tt

@@ -0,0 +1,24 @@
+class CreateEasyMLDeploys < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    create_table :easy_ml_deploys do |t|
+      t.bigint :model_id
+      t.bigint :model_history_id
+      t.bigint :retraining_run_id
+      t.bigint :model_file_id
+      t.string :status, null: false
+      t.string :trigger, default: 'manual'
+      t.text :stacktrace
+      t.string :snapshot_id
+      t.timestamps
+
+      t.index :created_at
+      t.index :model_id
+      t.index :model_history_id
+      t.index :snapshot_id
+      t.index :model_file_id
+      t.index :retraining_run_id
+      t.index :status
+      t.index :trigger
+    end
+  end
+end 

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_events.rb.tt

@@ -0,0 +1,20 @@
+class CreateEasyMLEvents < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    create_table :easy_ml_events do |t|
+      t.string :name, null: false
+      t.string :status, null: false
+      t.string :eventable_type
+      t.bigint :eventable_id
+      t.text :stacktrace
+
+      t.timestamps
+
+      t.index :name
+      t.index :status
+      t.index :eventable_type
+      t.index :eventable_id
+      t.index :created_at
+      t.index [:eventable_type, :eventable_id]
+    end
+  end
+end 

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_feature_histories.rb.tt

@@ -0,0 +1,14 @@
+require "historiographer/postgres_migration"
+
+class CreateEasyMLFeatureHistories < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    create_table :easy_ml_feature_histories do |t|
+      t.histories(
+        foreign_key: :feature_id,
+        index_names: {
+          [:dataset_id, :feature_position] => "idx_feature_histories_on_dataset_and_position"
+        }
+      )
+    end
+  end
+end 

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_features.rb.tt

@@ -0,0 +1,32 @@
+class CreateEasyMLFeatures < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    create_table :easy_ml_features do |t|
+      t.bigint :dataset_id, null: false
+      t.string :name
+      t.bigint :version
+      t.string :feature_class, null: false
+      t.integer :feature_position
+      t.integer :batch_size
+      t.boolean :needs_fit
+      t.string :sha
+      t.string :primary_key, array: true
+      t.datetime :applied_at
+      t.datetime :fit_at
+      t.bigint :refresh_every
+
+      t.timestamps
+
+      t.index %i[dataset_id feature_position], name: "idx_features_on_dataset_and_position"
+      t.index %i[dataset_id name], unique: true, name: "idx_features_on_dataset_and_name"
+      t.index :feature_class
+      t.index :applied_at
+      t.index :name
+      t.index :version
+      t.index :sha
+      t.index :batch_size
+      t.index :needs_fit
+      t.index :fit_at
+      t.index :refresh_every
+    end
+  end
+end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_model_file_histories.rb.tt

@@ -0,0 +1,9 @@
+require "historiographer/postgres_migration"
+
+class CreateEasyMLModelFileHistories < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    create_table :easy_ml_model_file_histories do |t|
+      t.histories(foreign_key: :model_file_id)
+    end
+  end
+end 

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_model_files.rb.tt

@@ -0,0 +1,17 @@
+class CreateEasyMLModelFiles < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    create_table :easy_ml_model_files do |t|
+      t.string :filename, null: false
+      t.string :path, null: false
+      t.json :configuration
+      t.string :model_type
+      t.bigint :model_id
+      t.timestamps
+
+      t.index :created_at
+      t.index :filename
+      t.index [:model_type]
+      t.index :model_id
+    end
+  end
+end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_model_histories.rb.tt

@@ -0,0 +1,9 @@
+require "historiographer/postgres_migration"
+
+class CreateEasyMLModelHistories < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    create_table :easy_ml_model_histories do |t|
+      t.histories(foreign_key: :model_id)
+    end
+  end
+end 

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_models.rb.tt

@@ -1,23 +1,34 @@
 # lib/railtie/generators/templates/migration/create_easy_ml_models.rb.tt
-class CreateEasyMLModels < ActiveRecord::Migration[6.0]
+class CreateEasyMLModels < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
   def change
     create_table :easy_ml_models do |t|
       t.string :name, null: false
-      t.boolean :is_live, default: false
+      t.string :model_type
+      t.string :status
+      t.bigint :dataset_id
+      t.bigint :model_file_id
+      t.json :configuration
       t.string :version, null: false
-      t.string :ml_model
-      t.string :task
-      t.json :metrics, default: []
-      t.json :file, null: false
+      t.string :root_dir
+      t.json :file
+      t.string :sha
+      t.datetime :last_trained_at
+      t.boolean :is_training
 
       t.timestamps
 
       t.index :created_at
+      t.index :last_trained_at
       t.index :name
       t.index :version
-      t.index :is_live
-      t.index [:name, :version], unique: true
-      t.index [:name, :version, :is_live]
+      t.index :status
+      t.index [:name, :status]
+      t.index [:name, :version]
+      t.index :dataset_id
+      t.index :model_type
+      t.index :model_file_id
+      t.index :sha
+      t.index :is_training
     end
   end
 end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_predictions.rb.tt

@@ -0,0 +1,17 @@
+class CreateEasyMLPredictions < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    create_table :easy_ml_predictions do |t|
+      t.bigint :model_id, null: false
+      t.bigint :model_history_id
+      t.string :prediction_type
+      t.jsonb :prediction_value
+      t.jsonb :raw_input
+      t.jsonb :normalized_input
+      t.timestamps
+
+      t.index :model_id
+      t.index :model_history_id
+      t.index :created_at
+    end
+  end
+end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_retraining_jobs.rb.tt

@@ -0,0 +1,77 @@
+class CreateEasyMLRetrainingJobs < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    create_table :easy_ml_retraining_jobs do |t|
+      t.bigint :model_id
+      t.string :frequency, null: false  # day, week, month, hour
+      t.json :at, null: false           # hour of day (0-23)
+      t.json :evaluator                 # Model evaluator
+      t.boolean :tuning_enabled, default: false
+      t.json :tuner_config              # configuration for the tuner
+      t.string :tuning_frequency        # day, week, month, hour - when to run with tuner
+      t.datetime :last_tuning_at        # track last tuning run
+      t.boolean :active, default: true
+      t.string :status, default: "pending"
+      t.datetime :last_run_at
+      t.string :metric, null: false
+      t.string :direction, null: false
+      t.float :threshold, null: false
+      t.boolean :auto_deploy, default: false
+      t.boolean :batch_mode
+      t.integer :batch_size
+      t.integer :batch_overlap
+      t.string :batch_key
+
+      t.timestamps
+
+      t.index :model_id
+      t.index :active
+      t.index :last_run_at
+      t.index :last_tuning_at
+      t.index :batch_mode
+      t.index :auto_deploy
+      t.index :tuning_enabled
+    end
+
+    create_table :easy_ml_retraining_runs do |t|
+      t.bigint :model_id
+      t.bigint :model_history_id
+      t.bigint :model_file_id
+      t.bigint :retraining_job_id, null: false
+      t.bigint :tuner_job_id, null: true
+      t.string :status, default: 'pending'
+      t.float :metric_value
+      t.float :threshold
+      t.string :trigger, default: 'manual'
+      t.string :threshold_direction
+      t.datetime :started_at
+      t.datetime :completed_at
+      t.text :error_message
+      t.jsonb :metadata
+      t.jsonb :metrics
+      t.jsonb :best_params
+      t.string :wandb_url
+      t.string :snapshot_id
+      t.boolean :deployable
+      t.boolean :is_deploying
+      t.boolean :deployed
+      t.bigint :deploy_id
+
+      t.timestamps
+
+      t.index :status
+      t.index :started_at
+      t.index :completed_at
+      t.index :created_at
+      t.index :tuner_job_id
+      t.index :retraining_job_id
+      t.index :model_id
+      t.index :trigger
+      t.index :wandb_url
+      t.index :snapshot_id
+      t.index :deploy_id
+      t.index :model_history_id
+      t.index :deployable
+      t.index :is_deploying
+    end
+  end
+end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_settings.rb.tt

@@ -0,0 +1,9 @@
+class CreateEasyMLSettings < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    create_table :easy_ml_settings do |t|
+      t.json :configuration
+
+      t.timestamps
+    end
+  end
+end 

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_splitter_histories.rb.tt

@@ -0,0 +1,9 @@
+require "historiographer/postgres_migration"
+
+class CreateEasyMLSplitterHistories < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    create_table :easy_ml_splitter_histories do |t|
+      t.histories(foreign_key: :splitter_id)
+    end
+  end
+end 

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_splitters.rb.tt

@@ -0,0 +1,15 @@
+class CreateEasyMLSplitters < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    create_table :easy_ml_splitters do |t|
+      t.string :splitter_type, null: false
+      t.json :configuration
+      t.bigint :dataset_id, null: false
+
+      t.timestamps
+
+      t.index :splitter_type
+      t.index :created_at
+      t.index :dataset_id
+    end
+  end
+end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_tuner_jobs.rb.tt

@@ -0,0 +1,40 @@
+class CreateEasyMLTunerJobs < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    create_table :easy_ml_tuner_jobs do |t|
+      t.json :config, null: false
+      t.bigint :best_tuner_run_id
+      t.bigint :model_id, null: false
+      t.string :status
+      t.string :direction, default: 'minimize'
+      t.datetime :started_at
+      t.datetime :completed_at
+      t.jsonb :metadata
+      t.string :wandb_url
+
+      t.timestamps
+
+      t.index :status
+      t.index :started_at
+      t.index :completed_at
+      t.index :model_id
+      t.index :best_tuner_run_id
+      t.index :wandb_url
+    end
+
+    create_table :easy_ml_tuner_runs do |t|
+      t.bigint :tuner_job_id, null: false
+      t.json :hyperparameters, null: false
+      t.float :value
+      t.integer :trial_number
+      t.string :status
+      t.string :wandb_url
+
+      t.timestamps
+
+      t.index [:tuner_job_id, :value]
+      t.index [:tuner_job_id, :trial_number], name: "idx_tuner_runs_and_trial_number"
+      t.index :status
+      t.index :wandb_url
+    end
+  end
+end

data/lib/easy_ml/support/est.rb

@@ -1 +1,5 @@
-EST = ActiveSupport::TimeZone.new("America/New_York")
+module EasyML
+  module Support
+    EST = ActiveSupport::TimeZone.new("America/New_York") unless defined?(EST)
+  end
+end

data/lib/easy_ml/support/file_rotate.rb

@@ -1,22 +1,86 @@
 module EasyML
-  class FileRotate
-    def initialize(directory, files_to_keep)
-      @directory = directory
-      @files_to_keep = files_to_keep
-    end
+  module Support
+    class FileRotate
+      def initialize(directory, files_to_keep, verbose: false)
+        @directory = directory
+        @files_to_keep = files_to_keep
+        @stats = { checked: 0, kept: 0, deleted: 0 }
+        @verbose = verbose
+      end
+
+      def cleanup(allowed_endings = [])
+        return unless @directory.present?
+
+        log "\nStarting file rotation in: #{@directory}"
+        log "Files to keep: #{@files_to_keep.count}"
+
+        process_directory(@directory, allowed_endings)
+        cleanup_empty_directories(@directory)
+
+        log "\nFile rotation complete:"
+        log "Files checked: #{@stats[:checked]}"
+        log "Files kept: #{@stats[:kept]}"
+        log "Files deleted: #{@stats[:deleted]}"
+      end
+
+      private
+
+      def process_directory(dir, allowed_endings)
+        return unless Dir.exist?(dir)
+
+        log "\nProcessing directory: #{dir}"
 
-    def cleanup(allowed_endings = %w[json])
-      return unless @directory.present?
+        if @files_to_keep.include?(dir)
+          log "  Keeping entire directory: #{dir}"
+          @stats[:kept] += 1
+          return
+        end
 
-      allowed_patterns = allowed_endings.map { |ending| File.join(@directory, "**", "*#{ending}") }
-      files_to_check = allowed_patterns.empty? ? Dir.glob(File.join(@directory, "**/*")) : Dir.glob(allowed_patterns)
-      # Filter out directories
-      files_to_check = files_to_check.select { |file| File.file?(file) }
+        allowed_patterns = allowed_endings.map { |ending| File.join(dir, "*#{ending}") }
+        files_to_check = allowed_patterns.empty? ? Dir.glob(File.join(dir, "*")) : Dir.glob(allowed_patterns)
+
+        files_to_check.each do |file|
+          next unless File.file?(file)
+
+          @stats[:checked] += 1
+
+          FileUtils.chown_R(`whoami`.chomp, "staff", file)
+          FileUtils.chmod_R(0o777, file)
+
+          if @files_to_keep.exclude?(file) && File.exist?(file)
+            log "  Deleting: #{file}"
+            File.delete(file)
+            @stats[:deleted] += 1
+          else
+            log "  Keeping: #{file}"
+            @stats[:kept] += 1
+          end
+        end
+
+        Dir.each_child(dir) do |child|
+          full_path = File.join(dir, child)
+          process_directory(full_path, allowed_endings) if File.directory?(full_path)
+        end
+      end
+
+      def cleanup_empty_directories(dir)
+        return unless Dir.exist?(dir)
+
+        Dir.each_child(dir) do |child|
+          full_path = File.join(dir, child)
+          cleanup_empty_directories(full_path) if File.directory?(full_path)
+        end
+
+        if Dir.empty?(dir) && dir != @directory
+          log "Removing empty directory: #{dir}"
+          Dir.rmdir(dir)
+        end
+      rescue Errno::ENOTEMPTY, Errno::EACCES
+        # Directory not empty or permission denied - skip it
+      end
 
-      files_to_check.each do |file|
-        FileUtils.chown_R(`whoami`.chomp, "staff", file)
-        FileUtils.chmod_R(0o777, file)
-        File.delete(file) if @files_to_keep.exclude?(file) && File.exist?(file)
+      def log(message)
+        puts message if @verbose
       end
     end
   end

data/lib/easy_ml/support/file_support.rb

@@ -0,0 +1,9 @@
+module EasyML
+  module Support
+    module FileSupport
+      def ensure_directory_exists(dir)
+        FileUtils.mkdir_p(dir) unless File.directory?(dir)
+      end
+    end
+  end
+end

data/lib/easy_ml/support/local_file.rb

@@ -0,0 +1,24 @@
+module EasyML
+  module Support
+    class LocalFile
+      attr_accessor :root_dir, :filename
+
+      def initialize(options = {})
+        @root_dir = options[:root_dir]
+        @filename = options[:filename]
+      end
+
+      def upload(file_path)
+        file_path
+      end
+
+      def download(full_path)
+        full_path
+      end
+
+      def synced?
+        true
+      end
+    end
+  end
+end

data/lib/easy_ml/support/lockable.rb

@@ -0,0 +1,62 @@
+module EasyML
+  module Support
+    module Lockable
+      KEYS_HASH = "easy_ml:lock_keys"
+
+      def self.unlock!(key)
+        suo_client = lock_client(key)
+
+        suo_client.locks.map(&:last).each do |lock_key|
+          suo_client.unlock(lock_key)
+        end
+      end
+
+      def self.locked?(key)
+        suo_client = lock_client(key)
+        suo_client.locked?
+      end
+
+      def self.locks(key)
+        suo_client = lock_client(key)
+        suo_client.locks
+      end
+
+      def self.lock_client(key, wait_timeout: 0.1, stale_timeout: 60 * 10, resources: 1)
+        Suo::Client::Redis.new(prefixed_key(key), {
+          acquisition_timeout: wait_timeout,
+          stale_lock_expiry: stale_timeout,
+          resources: resources,
+          client: client,
+        })
+      end
+
+      def self.prefixed_key(key)
+        "easy_ml:#{key}"
+      end
+
+      # Execute a block with a Redis lock
+      def self.with_lock(key, wait_timeout: 0.1, stale_timeout: 60 * 10, resources: 1)
+        lock_key = nil
+        suo_client = lock_client(key, wait_timeout: wait_timeout, stale_timeout: stale_timeout, resources: resources)
+        begin
+          lock_key = suo_client.lock
+          if lock_key
+            yield suo_client
+          end
+        ensure
+          suo_client.unlock(lock_key) if lock_key
+        end
+      end
+
+      # Redis client
+      def self.client
+        @client ||= Redis.new(host: redis_host)
+      end
+
+      # Determine Redis host
+      def self.redis_host
+        ENV["REDIS_HOST"] || (defined?(Resque) ? Resque.redis.client.host : "localhost")
+      end
+    end
+  end
+end