easy_ml 0.1.1

data/lib/easy_ml/data.rb

@@ -0,0 +1,8 @@
+module EasyML
+  module Data
+    require_relative "data/utils"
+    require_relative "data/preprocessor"
+    require_relative "data/dataset"
+    require_relative "data/datasource"
+  end
+end

data/lib/easy_ml/deployment.rb

@@ -0,0 +1,5 @@
+module EasyML
+  module Deployment
+    require_relative "deployment/model_uploader"
+  end
+end

data/lib/easy_ml/engine.rb

@@ -0,0 +1,26 @@
+require "rails/engine"
+
+module EasyML
+  class Engine < Rails::Engine
+    isolate_namespace EasyML
+
+    initializer "easy_ml.inflections" do
+      require_relative "initializers/inflections"
+    end
+
+    initializer "easy_ml.setup_generators" do |app|
+      app.config.generators do |g|
+        g.templates.unshift File.expand_path("../templates", __dir__)
+      end
+    end
+
+    generators_path = File.expand_path("railtie/generators", __dir__)
+    generators_dirs = Dir[File.join(generators_path, "**", "*.rb")]
+    generators_dirs.each { |file| require file }
+
+    config.after_initialize do
+      require_relative "../../app/models/easy_ml/model"
+      require_relative "../../app/models/easy_ml/models"
+    end
+  end
+end

data/lib/easy_ml/initializers/inflections.rb

@@ -0,0 +1,4 @@
+ActiveSupport::Inflector.inflections(:en) do |inflect|
+  inflect.acronym "EasyML"
+  inflect.acronym "ML"
+end

data/lib/easy_ml/logging.rb

@@ -0,0 +1,38 @@
+module EasyML
+  module Logging
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    module ClassMethods
+      def log_method(method_name, message, verbose: false)
+        original_method = instance_method(method_name)
+        define_method(method_name) do |*args, &block|
+          log_message(message, verbose: verbose)
+          result = original_method.bind(self).call(*args, &block)
+          result
+        end
+      end
+    end
+
+    def log_message(message, verbose: false)
+      if verbose
+        log_verbose(message)
+      else
+        puts message
+      end
+    end
+
+    def log_verbose(message)
+      puts message if @verbose
+    end
+
+    def log_warning(message)
+      puts "\e[33mWARNING: #{message}\e[0m"
+    end
+
+    def log_info(message)
+      puts "\e[34mINFO: #{message}\e[0m"
+    end
+  end
+end

data/lib/easy_ml/railtie/generators/migration/migration_generator.rb

@@ -0,0 +1,42 @@
+# lib/railtie/generators/migration/migration_generator.rb
+require "rails/generators"
+require "rails/generators/active_record/migration"
+
+module EasyML
+  module Generators
+    module Migration
+      class MigrationGenerator < Rails::Generators::Base
+        include Rails::Generators::Migration
+        namespace "easy_ml:migration"
+
+        # Set the source directory for templates
+        source_root File.expand_path("../../templates/migration", __dir__)
+
+        # Define the migration name
+        desc "Generates a migration for EasyMLModel with version and file for remote storage"
+
+        # Define the migration name; can be customized if needed
+        def self.migration_name
+          "create_easy_ml_models"
+        end
+
+        # Specify the next migration number
+        def self.next_migration_number(dirname)
+          if ActiveRecord.version < Gem::Version.new("7")
+            Time.now.utc.strftime("%Y%m%d%H%M%S")
+          elsif ActiveRecord.timestamped_migrations
+            Time.now.utc.strftime("%Y%m%d%H%M%S")
+          else
+            format("%.3d", (current_migration_number(dirname) + 1))
+          end
+        end
+
+        # Generate the migration file using the template
+        def create_migration_file
+          migration_template "create_easy_ml_models.rb.tt",
+                             "db/migrate/#{self.class.migration_name}.rb"
+        end
+      end
+    end
+  end
+end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_models.rb.tt

@@ -0,0 +1,23 @@
+# lib/railtie/generators/templates/migration/create_easy_ml_models.rb.tt
+class CreateEasyMLModels < ActiveRecord::Migration[6.0]
+  def change
+    create_table :easy_ml_models do |t|
+      t.string :name, null: false
+      t.boolean :is_live, default: false
+      t.string :version, null: false
+      t.string :ml_model
+      t.string :task
+      t.json :metrics, default: []
+      t.json :file, null: false
+
+      t.timestamps
+
+      t.index :created_at
+      t.index :name
+      t.index :version
+      t.index :is_live
+      t.index [:name, :version], unique: true
+      t.index [:name, :version, :is_live]
+    end
+  end
+end

data/lib/easy_ml/support/age.rb

@@ -0,0 +1,27 @@
+require "active_support/duration"
+
+module EasyML
+  module Support
+    module Age
+      def self.age(start_time, end_time, format: "human")
+        return nil unless start_time && end_time
+
+        age_duration = ActiveSupport::Duration.build((end_time - start_time).to_i)
+        age_parts = age_duration.parts
+
+        case format.to_s
+        when "human"
+          age_duration.inspect
+        when "days"
+          age_parts[:days]
+        when "hours"
+          age_parts[:hours]
+        when "minutes"
+          age_parts[:minutes]
+        when "integer"
+          age_duration.to_i
+        end
+      end
+    end
+  end
+end

data/lib/easy_ml/support/est.rb

@@ -0,0 +1 @@
+EST = ActiveSupport::TimeZone.new("America/New_York")

data/lib/easy_ml/support/file_rotate.rb

@@ -0,0 +1,23 @@
+module EasyML
+  class FileRotate
+    def initialize(directory, files_to_keep)
+      @directory = directory
+      @files_to_keep = files_to_keep
+    end
+
+    def cleanup(allowed_endings = %w[json])
+      return unless @directory.present?
+
+      allowed_patterns = allowed_endings.map { |ending| File.join(@directory, "**", "*#{ending}") }
+      files_to_check = allowed_patterns.empty? ? Dir.glob(File.join(@directory, "**/*")) : Dir.glob(allowed_patterns)
+      # Filter out directories
+      files_to_check = files_to_check.select { |file| File.file?(file) }
+
+      files_to_check.each do |file|
+        FileUtils.chown_R(`whoami`.chomp, "staff", file)
+        FileUtils.chmod_R(0o777, file)
+        File.delete(file) unless @files_to_keep.include?(file)
+      end
+    end
+  end
+end

data/lib/easy_ml/support/git_ignorable.rb

@@ -0,0 +1,66 @@
+require "active_support/concern"
+require "fileutils"
+
+module EasyML
+  module Support
+    module GitIgnorable
+      extend ActiveSupport::Concern
+
+      included do
+        class_attribute :gitignore_attributes, default: {}
+
+        def self.set_gitignore_callbacks(attribute, &block)
+          gitignore_attributes[attribute] = block
+
+          prepend GitignoreInitializer
+        end
+      end
+
+      module GitignoreInitializer
+        def initialize(options)
+          super
+          update_gitignore
+        end
+      end
+
+      class_methods do
+        def gitignore(attribute, &block)
+          set_gitignore_callbacks(attribute, &block)
+        end
+      end
+
+      def update_gitignore
+        self.class.gitignore_attributes.each do |attribute, block|
+          attribute_value = send(attribute)
+          next if attribute_value.blank?
+
+          patterns = block ? block.call(attribute_value) : attribute_value
+          next if patterns.nil? || (patterns.respond_to?(:empty?) && patterns.empty?)
+
+          patterns = [patterns] unless patterns.is_a?(Array)
+          patterns = relativize(patterns)
+          gitignore_path = File.join(Dir.pwd, ".gitignore")
+
+          FileUtils.mkdir_p(File.dirname(gitignore_path))
+          FileUtils.touch(gitignore_path) unless File.exist?(gitignore_path)
+
+          existing_content = File.read(gitignore_path).split("\n")
+          new_patterns = patterns.reject { |pattern| existing_content.include?(pattern) }
+          next if new_patterns.empty?
+
+          new_content = (existing_content + new_patterns).join("\n").strip
+          File.write(gitignore_path, new_content)
+        end
+      end
+
+      private
+
+      # Turn patterns like /Users/xyz/path/to/rails/x/**/* into: x/**/*
+      def relativize(patterns)
+        patterns.map do |pattern|
+          pattern.sub(%r{^#{Regexp.escape(Dir.pwd)}/}, "")
+        end
+      end
+    end
+  end
+end

data/lib/easy_ml/support/synced_directory.rb

@@ -0,0 +1,134 @@
+require "glue_gun"
+
+module EasyML
+  module Support
+    class SyncedDirectory
+      include GlueGun::DSL
+
+      attribute :root_dir, :string
+      attribute :s3_bucket, :string
+      attribute :s3_prefix, :string
+      attribute :s3_access_key_id, :string
+      attribute :s3_secret_access_key, :string
+
+      def sync
+        return false if synced?
+
+        mk_dir
+        clean_dir!
+        download
+        true
+      end
+
+      def files
+        Dir.glob(File.join(root_dir, File.join(s3_prefix, "*.csv")))
+      end
+
+      def age(format: "human")
+        Age.age(last_updated_at, EST.now, format: format)
+      end
+
+      def stale?
+        !synced?
+      end
+
+      def synced?
+        return @synced unless @synced.nil?
+
+        @synced = calculate_synced
+      end
+
+      def last_updated_at
+        return nil if files.empty?
+
+        files.map { |file| File.mtime(file) }.max.in_time_zone(EST)
+      end
+
+      private
+
+      def mk_dir
+        FileUtils.mkdir_p(root_dir)
+      end
+
+      def clean_dir!
+        FileUtils.rm_rf(root_dir)
+      end
+
+      def s3
+        @s3 ||= begin
+          credentials = Aws::Credentials.new(s3_access_key_id, s3_secret_access_key)
+          Aws::S3::Client.new(credentials: credentials)
+        end
+      end
+
+      def download
+        s3.list_objects_v2(bucket: s3_bucket, prefix: s3_prefix).contents.each do |object|
+          next if object.key.end_with?("/") # skip folders
+
+          gzipped_file_path = File.join(root_dir, object.key)
+          FileUtils.mkdir_p(File.dirname(gzipped_file_path))
+
+          s3.get_object(
+            response_target: gzipped_file_path,
+            bucket: s3_bucket,
+            key: object.key
+          )
+
+          puts "Downloaded #{object.key} to #{gzipped_file_path}"
+
+          # Ungzip the file
+          ungzipped_file_path = ungzip_file(gzipped_file_path)
+          puts "Ungzipped to #{ungzipped_file_path}"
+        end
+      end
+
+      def ungzip_file(gzipped_file_path)
+        ungzipped_file_path = gzipped_file_path.sub(/\.gz$/, "")
+
+        Zlib::GzipReader.open(gzipped_file_path) do |gz|
+          File.open(ungzipped_file_path, "wb") do |file|
+            file.write(gz.read)
+          end
+        end
+
+        File.delete(gzipped_file_path) # Optionally delete the gzipped file after extraction
+        ungzipped_file_path
+      end
+
+      def expand_dir(dir)
+        return dir if dir.to_s[0] == "/"
+
+        Rails.root.join(dir)
+      end
+
+      def new_data_available?
+        return true if files.empty?
+
+        local_latest = last_updated_at
+        s3_latest = s3_last_updated_at
+
+        return false if s3_latest.nil?
+
+        s3_latest > local_latest
+      end
+
+      def calculate_synced
+        return false if age.nil?
+
+        !new_data_available?
+      end
+
+      def s3_last_updated_at
+        s3_latest = nil
+
+        s3.list_objects_v2(bucket: s3_bucket, prefix: s3_prefix).contents.each do |object|
+          next if object.key.end_with?("/")
+
+          s3_latest = [s3_latest, object.last_modified].compact.max
+        end
+
+        s3_latest.in_time_zone(EST)
+      end
+    end
+  end
+end

data/lib/easy_ml/support/utc.rb

@@ -0,0 +1 @@
+UTC = ActiveSupport::TimeZone.new("UTC")

data/lib/easy_ml/support.rb

@@ -0,0 +1,10 @@
+module EasyML
+  module Support
+    require_relative "support/age"
+    require_relative "support/git_ignorable"
+    require_relative "support/synced_directory"
+    require_relative "support/utc"
+    require_relative "support/est"
+    require_relative "support/file_rotate"
+  end
+end

data/lib/easy_ml/trainer.rb

@@ -0,0 +1,92 @@
+module EasyML
+  class Trainer
+    # include GlueGun::DSL
+    # include EasyML::Logging
+
+    # define_attr :verbose, default: false
+    # define_attr :root_dir do |root_dir|
+    #   File.join(root_dir, "trainer")
+    # end
+
+    # define_config :dataset do |config|
+    #   config.define_option :default do |option|
+    #     option.set_class EasyML::Data::Dataset
+    #     option.define_attr :root_dir
+    #     option.define_attr :target
+    #     option.define_attr :batch_size
+    #   end
+    # end
+
+    # define_config :model do |config|
+    #   config.define_option :default do |option|
+    #     option.set_class EasyML::Model
+    #     option.define_attr :root_dir
+    #     option.define_attr :name
+    #     option.define_attr :hyperparameters
+    #   end
+    # end
+
+    # def train
+    #   log_info("Starting training process") if verbose
+
+    #   dataset.refresh!
+
+    #   log_info("Fitting model") if verbose
+    #   dataset.train(split_ys: true) do |xs, ys|
+    #     model.fit(xs, ys)
+    #   end
+
+    #   log_info("Saving model") if verbose
+    #   model.save
+
+    #   log_info("Training completed") if verbose
+    # end
+
+    # def evaluate
+    #   log_info("Starting evaluation process") if verbose
+
+    #   results = {}
+
+    #   %i[train test valid].each do |split|
+    #     log_info("Evaluating on #{split} set") if verbose
+    #     predictions = []
+    #     actuals = []
+
+    #     dataset.send(split, split_ys: true) do |xs, ys|
+    #       batch_predictions = model.predict(xs)
+    #       predictions.concat(batch_predictions.to_a)
+    #       actuals.concat(ys.to_a)
+    #     end
+
+    #     results[split] = calculate_metrics(predictions, actuals)
+    #   end
+
+    #   log_info("Evaluation completed") if verbose
+    #   results
+    # end
+
+    # private
+
+    # def calculate_metrics(predictions, actuals)
+    #   # Implement your metric calculations here
+    #   # This is a placeholder and should be replaced with actual metric calculations
+    #   {
+    #     mse: mean_squared_error(predictions, actuals),
+    #     mae: mean_absolute_error(predictions, actuals),
+    #     r2: r_squared(predictions, actuals)
+    #   }
+    # end
+
+    # def mean_squared_error(predictions, actuals)
+    #   # Implement MSE calculation
+    # end
+
+    # def mean_absolute_error(predictions, actuals)
+    #   # Implement MAE calculation
+    # end
+
+    # def r_squared(predictions, actuals)
+    #   # Implement R-squared calculation
+    # end
+  end
+end

data/lib/easy_ml/transforms.rb

@@ -0,0 +1,29 @@
+module EasyML::Transforms
+  def self.included(base)
+    base.extend(ClassMethods)
+  end
+
+  module ClassMethods
+    def transforms
+      @transforms ||= []
+    end
+
+    def transform(method_name)
+      transforms << method_name
+    end
+
+    def apply_transforms(df)
+      new.apply_transforms(df)
+    end
+  end
+
+  def missing_any?(list1, list2)
+    (list1 - list2).any?
+  end
+
+  def apply_transforms(df)
+    self.class.transforms.reduce(df) do |df, transform_method|
+      send(transform_method, df)
+    end
+  end
+end

data/lib/easy_ml/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module EasyML
+  VERSION = "0.1.1"
+end

data/lib/easy_ml.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "rails"
+require "active_record"
+require "active_model"
+require "active_support/all"
+require "glue_gun"
+require "numo/narray"
+require "xgboost"
+require_relative "easy_ml/version"
+require_relative "easy_ml/engine"
+
+module EasyML
+  class Error < StandardError; end
+
+  require_relative "easy_ml/support"
+  require_relative "easy_ml/core_ext"
+  require_relative "easy_ml/logging"
+  require_relative "easy_ml/data"
+  require_relative "easy_ml/transforms"
+  require_relative "easy_ml/core"
+  require_relative "easy_ml/trainer"
+end