easy_ml 0.1.4 → 0.2.0.pre.rc1

data/lib/easy_ml/data/statistics_learner.rb

@@ -0,0 +1,93 @@
+require_relative "date_converter"
+require_relative "polars_column"
+
+module EasyML::Data
+  class StatisticsLearner
+    attr_accessor :verbose
+
+    def initialize(options = {})
+      @verbose = options[:verbose]
+    end
+
+    def self.learn(raw, processed)
+      output = { raw: learn_split(raw) }
+      output[:processed] = learn_split(processed) if processed.data.present?
+      output
+    end
+
+    def self.learn_split(split)
+      df = split.read(:all)
+      train_df = split.read(:train)
+      all_stats = learn_df(df)
+      train_stats = learn_df(train_df)
+
+      all_stats.reduce({}) do |output, (k, _)|
+        output.tap do
+          output[k] = all_stats[k].slice(:num_rows, :null_count, :unique_count, :counts).merge!(
+            train_stats[k].slice(:mean, :median, :min, :max, :std, :last_value, :most_frequent_value)
+          )
+        end
+      end
+    end
+
+    def self.learn_df(df)
+      return if df.nil?
+
+      base_stats = describe_to_h(df).deep_symbolize_keys
+
+      # Add basic column statistics first
+      df.columns.each_with_object({}) do |col, stats|
+        series = df[col]
+        return {} if series.dtype == Polars::Null
+        field_type = PolarsColumn.determine_type(series)
+
+        stats[col] = {
+          num_rows: series.shape,
+          null_count: base_stats[col.to_sym][:null_count].to_i,
+        }
+
+        # Add type-specific statistics
+        case field_type
+        when :integer, :float
+          allowed_attrs = if id_column?(col)
+              %i[field_type null_count min max]
+            else
+              base_stats[col.to_sym].keys
+            end
+          stats[col].merge!(base_stats[col.to_sym].slice(*allowed_attrs))
+        when :categorical, :string, :text, :boolean
+          stats[col].merge!(most_frequent_value: series.mode.sort.to_a&.first)
+          if field_type == :categorical
+            stats[col].merge!(
+              unique_count: series.n_unique,
+              counts: Hash[series.value_counts.to_hashes.map(&:values)],
+            )
+          end
+        when :datetime
+          stats[col].merge!(
+            unique_count: series.n_unique,
+            last_value: series.sort[-1],
+          )
+        end
+      end
+    end
+
+    def self.id_column?(column)
+      col = column.to_s.downcase
+      col.match?(/^id$/) || col.match?(/.*_id/)
+    end
+
+    def self.describe_to_h(df)
+      init_h = df.describe.to_h
+      rows = init_h.values.map(&:to_a)
+      keys = rows.first
+      column_names = init_h.keys[1..-1]
+      column_values = rows[1..-1]
+      column_names.zip(column_values).inject({}) do |hash, (col_name, col_values)|
+        hash.tap do
+          hash[col_name] = Hash[keys.zip(col_values)]
+        end
+      end
+    end
+  end
+end

data/lib/easy_ml/data/synced_directory.rb

@@ -0,0 +1,341 @@
+require_relative "polars_reader"
+
+module EasyML
+  module Data
+    class SyncedDirectory
+      attr_accessor :root_dir, :s3_bucket, :s3_prefix, :s3_access_key_id, :s3_secret_access_key, :cache_for, :polars_args
+
+      def initialize(options = {})
+        @root_dir = options.dig(:root_dir)
+        @s3_bucket = options.dig(:s3_bucket)
+        @s3_prefix = options.dig(:s3_prefix)
+        @s3_access_key_id = options.dig(:s3_access_key_id)
+        @s3_secret_access_key = options.dig(:s3_secret_access_key)
+        @cache_for = options.dig(:cache_for)
+        @polars_args = options.dig(:polars_args)
+      end
+
+      delegate :query, :data, :all_files, :files, to: :reader
+
+      def before_sync
+        return unless should_sync?
+
+        clean
+      end
+
+      def after_sync
+        reader.normalize
+      end
+
+      def clean
+        mk_dir
+        clean_dir!
+        reader.clean
+      end
+
+      def remote_files
+        s3.list_objects_v2(bucket: s3_bucket, prefix: s3_prefix)
+      end
+
+      def should_sync?(force = false)
+        force || !synced?
+      end
+
+      def sync!(parallel: true)
+        sync(force: true, parallel: parallel)
+      end
+
+      def sync(force: false, parallel: false)
+        return false unless should_sync?(force)
+
+        files = files_to_sync
+
+        if parallel
+          Parallel.each(files, in_processes: 4, timeout: 10) { |object| download_file(object) }
+        else
+          files.each { |object| download_file(object) }
+        end
+        true
+      end
+
+      def files_to_sync
+        objects = s3.list_objects_v2(bucket: s3_bucket, prefix: s3_prefix).contents
+        objects.reject { |object| object.key.end_with?("/") }
+      end
+
+      def in_batches(&block)
+        reader.in_batches(&block)
+      end
+
+      def files
+        reader.files
+      end
+
+      def age(format: "human")
+        EasyML::Support::Age.age(last_updated_at, EasyML::Support::EST.now, format: format)
+      end
+
+      def stale?
+        !synced?
+      end
+
+      def synced?
+        return @synced unless @synced.nil?
+
+        return true if use_cached?
+
+        @synced = calculate_synced
+      end
+
+      def use_cached?
+        return false unless cache_for.present?
+        return false if last_updated_at.nil?
+
+        age_in_seconds = EasyML::Support::Age.age(last_updated_at, EasyML::Support::EST.now, format: "integer")
+        age_in_seconds < cache_for.to_i
+      end
+
+      def last_updated_at
+        return nil if files.empty?
+
+        files.map { |file| File.mtime(file) }.max.in_time_zone(EasyML::Support::EST)
+      end
+
+      def schema
+        reader.schema
+      end
+
+      def num_rows
+        reader.num_rows
+      end
+
+      def download_file(object)
+        # When s3_prefix is present, strip it from the key and just use the filename
+        key_without_prefix = s3_prefix.present? ? object.key.sub(/^#{Regexp.escape(s3_prefix)}\//, "") : object.key
+        local_file_path = File.join(root_dir, File.basename(key_without_prefix))
+        FileUtils.mkdir_p(File.dirname(local_file_path))
+
+        Rails.logger.info("Downloading object #{object.key} to #{local_file_path}")
+
+        s3.get_object(
+          response_target: local_file_path,
+          bucket: s3_bucket,
+          key: object.key,
+        )
+
+        Rails.logger.info("Downloaded #{object.key} to #{local_file_path}")
+        ungzipped_file_path = ungzip_file(local_file_path)
+        Rails.logger.info("Ungzipped to #{ungzipped_file_path}")
+      rescue Aws::S3::Errors::ServiceError, Net::OpenTimeout, Net::ReadTimeout, StandardError => e
+        Rails.logger.error("Failed to process #{object.key}: #{e.message}")
+        raise e
+      end
+
+      def upload!(parallel: true)
+        upload(force: true, parallel: parallel)
+      end
+
+      def upload(force: false, parallel: true)
+        files = force ? files_to_upload : files_to_upload.select { |f| should_upload?(f) }
+        return true if files.empty?
+
+        if parallel
+          Parallel.each(files, in_processes: 4, timeout: 10) { |file| upload_file(file) }
+        else
+          files.each { |file| upload_file(file) }
+        end
+        true
+      end
+
+      def files_to_upload
+        return [] unless Dir.exist?(root_dir)
+
+        local_files = Dir.glob(File.join(root_dir, "**", "*")).select { |f| File.file?(f) }
+
+        # Get remote files and their last modified times
+        remote_files = {}
+        self.remote_files.contents.each do |object|
+          next if object.key.end_with?("/")
+
+          # Remove .gz extension and s3_prefix to match local paths
+          local_key = object.key.sub(/\.gz$/, "")
+          local_key = local_key.sub(%r{^#{Regexp.escape(s3_prefix)}/}, "") if s3_prefix.present?
+          remote_files[local_key] = object.last_modified.in_time_zone(EasyML::Support::EST)
+        end
+
+        # Filter files that are newer locally
+        local_files.select do |file_path|
+          relative_path = Pathname.new(file_path).relative_path_from(Pathname.new(root_dir)).to_s
+          local_mtime = File.mtime(file_path).in_time_zone(EasyML::Support::EST)
+
+          # Upload if file doesn't exist remotely or is newer locally
+          !remote_files.key?(relative_path) || local_mtime > remote_files[relative_path]
+        end
+      end
+
+      # Add aliases for sync methods
+      alias download! sync!
+      alias download sync
+
+      private
+
+      def dir
+        root_dir
+      end
+
+      def relative_path(path)
+        if s3_prefix.present?
+          path.sub(Regexp.escape(s3_prefix), "").gsub(%r{/$}, "")
+        else
+          path
+        end
+      end
+
+      def reader
+        return @reader if @reader
+
+        @reader = EasyML::Data::PolarsReader.new(
+          root_dir: dir,
+          polars_args: polars_args,
+          refresh: false,
+        )
+      end
+
+      def mk_dir
+        FileUtils.mkdir_p(root_dir)
+      end
+
+      def clean_dir!
+        unless root_dir.start_with?(Rails.root.to_s)
+          raise "Refusing to wipe directory #{root_dir}, as it is not in the scope of #{Rails.root}"
+        end
+
+        FileUtils.rm_rf(root_dir)
+      end
+
+      def s3
+        credentials = Aws::Credentials.new(
+          s3_access_key_id,
+          s3_secret_access_key
+        )
+        Aws::S3::Client.new(
+          credentials: credentials,
+          http_open_timeout: 5, # Timeout for establishing connection (in seconds)
+          http_read_timeout: 30, # Timeout for reading response (in seconds))
+          http_wire_trace: false, # Enable verbose HTTP logging
+          http_idle_timeout: 0,
+          logger: Logger.new(STDOUT), # Logs to STDOUT; you can also set a file
+        )
+      end
+
+      def ungzip_file(gzipped_file_path)
+        ungzipped_file_path = gzipped_file_path.sub(/\.gz$/, "")
+
+        Zlib::GzipReader.open(gzipped_file_path) do |gz|
+          File.open(ungzipped_file_path, "wb") do |file|
+            file.write(gz.read)
+          end
+        end
+
+        File.delete(gzipped_file_path) # Optionally delete the gzipped file after extraction
+        ungzipped_file_path
+      end
+
+      def expand_dir(dir)
+        return dir if dir.to_s[0] == "/"
+
+        Rails.root.join(dir)
+      end
+
+      def new_data_available?
+        return false if files_to_sync.empty?
+        return true if files.empty?
+
+        local_latest = last_updated_at
+        s3_latest = s3_last_updated_at
+
+        return false if s3_latest.nil?
+
+        s3_latest > local_latest
+      end
+
+      def calculate_synced
+        !new_data_available?
+      end
+
+      def s3_last_updated_at
+        s3_latest = nil
+
+        s3.list_objects_v2(bucket: s3_bucket, prefix: s3_prefix).contents.each do |object|
+          next if object.key.end_with?("/")
+
+          s3_latest = [s3_latest, object.last_modified].compact.max
+        end
+
+        s3_latest.in_time_zone(EasyML::Support::EST)
+      end
+
+      def upload_file(file_path)
+        relative_path = Pathname.new(file_path).relative_path_from(Pathname.new(root_dir)).to_s
+        s3_key = s3_prefix.present? ? File.join(s3_prefix, File.basename(relative_path)) : relative_path
+
+        # Create a temporary gzipped version of the file
+        gzipped_file_path = "#{file_path}.gz"
+
+        begin
+          Rails.logger.info("Compressing and uploading #{file_path} to s3://#{s3_bucket}/#{s3_key}")
+
+          # Compress the file
+          Zlib::GzipWriter.open(gzipped_file_path) do |gz|
+            File.open(file_path, "rb") do |file|
+              gz.write(file.read)
+            end
+          end
+
+          # Upload the gzipped file
+          File.open(gzipped_file_path, "rb") do |file|
+            s3.put_object(
+              bucket: s3_bucket,
+              key: "#{s3_key}.gz",
+              body: file,
+              content_encoding: "gzip",
+            )
+          end
+
+          Rails.logger.info("Successfully uploaded #{file_path} to s3://#{s3_bucket}/#{s3_key}.gz")
+        rescue Aws::S3::Errors::ServiceError, StandardError => e
+          Rails.logger.error("Failed to upload #{file_path}: #{e.message}")
+          raise e
+        ensure
+          # Clean up temporary gzipped file
+          File.delete(gzipped_file_path) if File.exist?(gzipped_file_path)
+        end
+      end
+
+      def should_upload?(file_path)
+        relative_path = Pathname.new(file_path).relative_path_from(Pathname.new(root_dir)).to_s
+        s3_key = s3_prefix.present? ? File.join(s3_prefix, relative_path) : relative_path
+
+        begin
+          # Check if file exists in S3
+          response = s3.head_object(
+            bucket: s3_bucket,
+            key: "#{s3_key}.gz",
+          )
+
+          # Compare modification times
+          local_mtime = File.mtime(file_path).in_time_zone(EasyML::Support::EST)
+          remote_mtime = response.last_modified.in_time_zone(EasyML::Support::EST)
+
+          local_mtime > remote_mtime
+        rescue Aws::S3::Errors::NotFound
+          # File doesn't exist in S3, should upload
+          true
+        rescue Aws::S3::Errors::ServiceError => e
+          Rails.logger.error("Error checking S3 object: #{e.message}")
+          raise e
+        end
+      end
+    end
+  end
+end

data/lib/easy_ml/data.rb

@@ -1,8 +1,12 @@
 module EasyML
   module Data
     require_relative "data/utils"
+    require_relative "data/polars_reader"
+    require_relative "data/synced_directory"
     require_relative "data/preprocessor"
-    require_relative "data/dataset"
-    require_relative "data/datasource"
+    require_relative "data/splits"
+    require_relative "data/polars_column"
+    require_relative "data/statistics_learner"
+    require_relative "data/date_converter"
   end
 end

data/lib/easy_ml/engine.rb

@@ -1,26 +1,125 @@
+require "aws-sdk"
+require "awesome_print"
+require "action_controller"
+require "inertia_rails"
+require "jsonapi/serializer"
+require "numo/narray"
+require "numpy"
+require "parallel"
+require "polars-df"
+require "pycall"
+require "optuna"
+require "tailwindcss-rails"
+require "wandb"
+require "xgb"
+require "sidekiq"
+require "vite_ruby"
 require "rails/engine"
+require "activerecord-import"
+require "historiographer"
 
 module EasyML
   class Engine < Rails::Engine
     isolate_namespace EasyML
 
+    def root_dir
+      Rails.root.join("easy_ml")
+    end
+
+    config.autoload_paths += [
+      root.join("app/models"),
+      root.join("app/models/datasources"),
+      root.join("app/models/models"),
+      root.join("lib/easy_ml"),
+    ]
+
+    config.eager_load_paths += [
+      root.join("app/models"),
+      root.join("app/models/datasources"),
+      root.join("app/models/models"),
+      root.join("lib/easy_ml"),
+    ]
+
     initializer "easy_ml.inflections" do
       require_relative "initializers/inflections"
+      EasyML::Initializers::Inflections.inflect
+    end
+
+    initializer "easy_ml.enable_string_cache" do
+      Polars.enable_string_cache
+    end
+
+    unless %w[rake rails].include?(File.basename($0)) && %w[generate db:migrate db:drop easy_ml:migration].include?(ARGV.first)
+      config.after_initialize do
+        Dir.glob(File.expand_path("app/models/easy_ml/datasources/*.rb", EasyML::Engine.root)).each do |file|
+          require file
+        end
+        Dir.glob(File.expand_path("app/models/easy_ml/models/*.rb", EasyML::Engine.root)).each do |file|
+          require file
+        end
+        Dir.glob(File.expand_path("app/models/easy_ml/splitters/*.rb", EasyML::Engine.root)).each do |file|
+          require file
+        end
+        Dir.glob(File.expand_path("app/models/easy_ml/**/*.rb", EasyML::Engine.root)).each do |file|
+          require file
+        end
+      end
+    end
+
+    initializer "easy_ml.active_job_config" do
+      ActiveSupport.on_load(:active_job) do
+        self.queue_adapter = :resque
+      end
+    end
+
+    # This tells our demo app where to look for assets like css, js
+    initializer "easy_ml.assets" do |app|
+      if app.config.respond_to?(:assets)
+        app.config.assets.precompile += %w[
+          easy_ml/application.js
+          easy_ml/application.css
+        ]
+        app.config.assets.paths << root.join("app", "frontend")
+      end
     end
 
     initializer "easy_ml.setup_generators" do |app|
+      generators_path = EasyML::Engine.root.join("lib/easy_ml/railtie/generators")
+      generators_dirs = Dir[File.join(generators_path, "**", "*.rb")]
+      generators_dirs.each { |file| require file }
+
       app.config.generators do |g|
         g.templates.unshift File.expand_path("../templates", __dir__)
       end
     end
 
-    generators_path = File.expand_path("railtie/generators", __dir__)
-    generators_dirs = Dir[File.join(generators_path, "**", "*.rb")]
-    generators_dirs.each { |file| require file }
+    delegate :vite_ruby, to: :class
+
+    def self.vite_ruby
+      @vite_ruby ||= ViteRuby.new(root: root)
+    end
+
+    unless Rails.env.development?
+      config.app_middleware.use(Rack::Static,
+                                urls: ["/#{vite_ruby.config.public_output_dir}"],
+                                root: root.join(vite_ruby.config.public_dir))
+    end
+
+    initializer "vite_rails_engine.proxy" do |app|
+      if vite_ruby.run_proxy?
+        # Use Vite proxy in development for live assets
+        app.middleware.insert_before 0, ViteRuby::DevServerProxy, ssl_verify_none: true, vite_ruby: vite_ruby
+      end
+    end
+
+    initializer "vite_rails_engine.logger" do
+      config.after_initialize do
+        vite_ruby.logger = Rails.logger
+      end
+    end
 
-    config.after_initialize do
-      require_relative "../../app/models/easy_ml/model"
-      require_relative "../../app/models/easy_ml/models"
+    def list_routes
+      EasyML::Engine.routes.routes.map { |r| "#{r.name} #{r.path.spec}" }
     end
   end
 end