easy_ml 0.1.4 → 0.2.0.pre.rc1

data/lib/easy_ml/support/synced_file.rb

@@ -0,0 +1,103 @@
+require "aws-sdk-s3"
+
+module EasyML
+  module Support
+    class SyncedFile
+      attr_accessor :root_dir, :filename, :s3_bucket, :s3_prefix,
+                    :s3_access_key_id, :s3_secret_access_key, :s3_region
+
+      def initialize(options = {})
+        @root_dir = options[:root_dir]
+        @filename = options[:filename]
+        @s3_bucket = options[:s3_bucket] || EasyML::Configuration.s3_bucket
+        @s3_prefix = options[:s3_prefix]
+        @s3_access_key_id = options[:s3_access_key_id] || EasyML::Configuration.s3_access_key_id
+        @s3_secret_access_key = options[:s3_secret_access_key] || EasyML::Configuration.s3_secret_access_key
+        @s3_region = options[:s3_region] || EasyML::Configuration.s3_region
+      end
+
+      # root_dir + s3_prefix = path
+      # path + filename = full_path
+      # input_file => "/Users/brettshollenberger/programming/easy_ml/lib/easy_ml/core/models/easy_ml_models/bart/xgboost_20241025120809.json"
+      # output_file => "/Users/brettshollenberger/programming/fundera/app/services/bart/standard/easy_ml_models/Bart/xgboost_20241025120809.json"
+      #
+      def upload(file_path)
+        # Ensure the file exists locally before attempting upload
+        raise "File #{file_path} does not exist" unless File.exist?(file_path)
+
+        # Calculate the path for the file on S3
+        self.filename = Pathname.new(file_path).basename.to_s
+        s3_key = s3_prefix.present? ? File.join(s3_prefix, filename) : filename
+
+        # Perform the upload to S3
+        s3.put_object(
+          bucket: s3_bucket,
+          key: s3_key,
+          body: File.open(file_path),
+        )
+
+        file_path
+      end
+
+      def download(full_path)
+        base_path = File.join(s3_prefix, filename)
+        FileUtils.mkdir_p(File.dirname(full_path))
+
+        s3.get_object(
+          response_target: full_path,
+          bucket: s3_bucket,
+          key: base_path,
+        )
+
+        full_path
+      end
+
+      def path
+        File.join(root_dir, s3_prefix)
+      end
+
+      def age(format: "human")
+        Age.age(last_updated_at, EasyML::Support::EST.now, format: format)
+      end
+
+      def stale?
+        !synced?
+      end
+
+      def synced?
+        return @synced unless @synced.nil?
+
+        return true if use_cached?
+
+        @synced = calculate_synced
+      end
+
+      def last_updated_at
+        return nil if files.empty?
+
+        files.map { |file| File.mtime(file) }.max.in_time_zone(EasyML::Support::EST)
+      end
+
+      private
+
+      def mk_dir
+        FileUtils.mkdir_p(root_dir)
+      end
+
+      def clean_dir!
+        unless root_dir.start_with?(Rails.root.to_s)
+          raise "Refusing to wipe directory #{root_dir}, as it is not in the scope of #{Rails.root}"
+        end
+
+        FileUtils.rm_rf(root_dir)
+      end
+
+      def s3
+        @s3 ||= begin
+            credentials = Aws::Credentials.new(s3_access_key_id, s3_secret_access_key)
+            Aws::S3::Client.new(credentials: credentials)
+          end
+      end
+    end
+  end
+end

data/lib/easy_ml/support/utc.rb

@@ -1 +1,5 @@
-UTC = ActiveSupport::TimeZone.new("UTC")
+module EasyML
+  module Support
+    UTC = ActiveSupport::TimeZone.new("UTC")
+  end
+end

data/lib/easy_ml/support.rb

@@ -1,10 +1,13 @@
 module EasyML
   module Support
-    require_relative "support/age"
-    require_relative "support/git_ignorable"
-    require_relative "support/synced_directory"
     require_relative "support/utc"
     require_relative "support/est"
+    require_relative "support/age"
+    require_relative "support/git_ignorable"
+    require_relative "support/synced_file"
+    require_relative "support/local_file"
     require_relative "support/file_rotate"
+    require_relative "support/file_support"
+    require_relative "support/lockable"
   end
 end

data/lib/easy_ml/version.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
 module EasyML
-  VERSION = "0.1.4"
+  VERSION = "0.2.0-rc1"
+
+  module Version
+  end
 end

data/lib/easy_ml.rb

@@ -4,19 +4,24 @@ require "rails"
 require "active_record"
 require "active_model"
 require "active_support/all"
-require "glue_gun"
 require "numo/narray"
 require "xgboost"
+require "suo"
+require "polars"
 require_relative "easy_ml/version"
 require_relative "easy_ml/engine"
 
 module EasyML
   class Error < StandardError; end
 
+  require_relative "easy_ml/configuration"
   require_relative "easy_ml/support"
   require_relative "easy_ml/core_ext"
   require_relative "easy_ml/logging"
   require_relative "easy_ml/data"
-  require_relative "easy_ml/transforms"
+  require_relative "easy_ml/data/filter_extensions"
+  require_relative "easy_ml/features"
+  require_relative "easy_ml/feature_store"
   require_relative "easy_ml/core"
+  require_relative "easy_ml/predict"
 end