easy_ml 0.2.0.pre.rc100 → 0.2.0.pre.rc101

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 60b022cce39806029405dad71468dfbe0ccac88182c11f224ff764aa3ac8de0c
-  data.tar.gz: '029679b95dacfc6009575467031bf5aa95d95d964ddd2e5fe88109a56183ad78'
+  metadata.gz: 7f48937aea567de8e40bc34486c4ac945b860ca26654d8d3b06efa1c1d4a54f3
+  data.tar.gz: 1abb8bb2e3f3ba8bb9c228d7a9691e8906ababa523ad0d7155cdafbd3ec62396
 SHA512:
-  metadata.gz: f3b147c4b3ed9f964742dab5dd6c0bea3d9375b4511dab02f71bc2f12ba28481d08928fe1d500446b79f19e542bc4d73457cc2e3144835ba5a2126c75a2feba1
-  data.tar.gz: 2406d8925786f4d9ebabd126af836f3aad01e089215b2a649e5f211584c660f58d0c293cd60d529646019389a64411d6e198ea5f7d1c962a15a37ece43b5c6ec
+  metadata.gz: ef28fcb989d2934329e4da3c9a138d3fc7b4c9ae995d7ce021217f4507e24b17664d2dee9690b7a00105e804b41f490442e2b90f1d76f8c12d7ddca768ae43ba
+  data.tar.gz: 3ba6f95ca3a660540e81a49c5eba84f530b606d89dc499e05aa288d26b90802dfc74b0b7615360002f1f36c1255f41ea6871378ffa1e93abc147b4f2a5c6ab0c

data/config/initializers/zhong.rb

@@ -8,6 +8,12 @@ if %w[zhong:start].include?(ARGV.first)
       EasyML::CleanJob.perform_later
     end
 
+    every 1.day, "refresh datasets" do
+      EasyML::Dataset.all.each do |dataset|
+        dataset.refresh_async
+      end
+    end
+
     every 1.hour, "schedule retraining" do
       EasyML::ScheduleRetrainingJob.perform_later
     end

data/lib/easy_ml/data/polars_column.rb

@@ -16,6 +16,7 @@ module EasyML
         array: Polars::List,
       }
       POLARS_MAP = {
+        Polars::Decimal => :float,
         Polars::Float64 => :float,
         Polars::Int64 => :integer,
         Polars::Float32 => :float,

data/lib/easy_ml/data/synced_directory.rb

@@ -126,7 +126,7 @@ module EasyML
         )
 
         Rails.logger.info("Downloaded #{object.key} to #{local_file_path}")
-        if object.key.end_with?(".gz")
+        if object.key.end_with?(".gz") && !object.key.end_with?(".parquet.gz")
           ungzipped_file_path = ungzip_file(local_file_path)
           Rails.logger.info("Ungzipped to #{ungzipped_file_path}")
         end
@@ -284,48 +284,61 @@ module EasyML
         relative_path = Pathname.new(file_path).relative_path_from(Pathname.new(root_dir)).to_s
         s3_key = s3_prefix.present? ? File.join(s3_prefix, File.basename(relative_path)) : relative_path
 
-        # Create a temporary gzipped version of the file
-        gzipped_file_path = "#{file_path}.gz"
-
         begin
-          Rails.logger.info("Compressing and uploading #{file_path} to s3://#{s3_bucket}/#{s3_key}")
+          Rails.logger.info("Uploading #{file_path} to s3://#{s3_bucket}/#{s3_key}")
 
-          # Compress the file
-          Zlib::GzipWriter.open(gzipped_file_path) do |gz|
+          if file_path.end_with?(".parquet")
+            # Upload parquet files directly without compression
             File.open(file_path, "rb") do |file|
-              gz.write(file.read)
+              s3.put_object(
+                bucket: s3_bucket,
+                key: s3_key,
+                body: file
+              )
+            end
+            Rails.logger.info("Successfully uploaded #{file_path} to s3://#{s3_bucket}/#{s3_key}")
+          else
+            # Create a temporary gzipped version of the file
+            gzipped_file_path = "#{file_path}.gz"
+
+            # Compress the file
+            Zlib::GzipWriter.open(gzipped_file_path) do |gz|
+              File.open(file_path, "rb") do |file|
+                gz.write(file.read)
+              end
             end
-          end
 
-          # Upload the gzipped file
-          File.open(gzipped_file_path, "rb") do |file|
-            s3.put_object(
-              bucket: s3_bucket,
-              key: "#{s3_key}.gz",
-              body: file,
-              content_encoding: "gzip",
-            )
-          end
+            # Upload the gzipped file
+            File.open(gzipped_file_path, "rb") do |file|
+              s3.put_object(
+                bucket: s3_bucket,
+                key: "#{s3_key}.gz",
+                body: file,
+                content_encoding: "gzip",
+              )
+            end
+
+            Rails.logger.info("Successfully uploaded #{file_path} to s3://#{s3_bucket}/#{s3_key}.gz")
 
-          Rails.logger.info("Successfully uploaded #{file_path} to s3://#{s3_bucket}/#{s3_key}.gz")
+            # Clean up temporary gzipped file
+            File.delete(gzipped_file_path) if File.exist?(gzipped_file_path)
+          end
         rescue Aws::S3::Errors::ServiceError, StandardError => e
           Rails.logger.error("Failed to upload #{file_path}: #{e.message}")
           raise e
-        ensure
-          # Clean up temporary gzipped file
-          File.delete(gzipped_file_path) if File.exist?(gzipped_file_path)
         end
       end
 
       def should_upload?(file_path)
         relative_path = Pathname.new(file_path).relative_path_from(Pathname.new(root_dir)).to_s
         s3_key = s3_prefix.present? ? File.join(s3_prefix, relative_path) : relative_path
+        s3_key = "#{s3_key}.gz" unless file_path.end_with?(".parquet")
 
         begin
           # Check if file exists in S3
           response = s3.head_object(
             bucket: s3_bucket,
-            key: "#{s3_key}.gz",
+            key: s3_key,
           )
 
           # Compare modification times

data/lib/easy_ml/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module EasyML
-  VERSION = "0.2.0-rc100"
+  VERSION = "0.2.0-rc101"
 
   module Version
   end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: easy_ml
 version: !ruby/object:Gem::Version
-  version: 0.2.0.pre.rc100
+  version: 0.2.0.pre.rc101
 platform: ruby
 authors:
 - Brett Shollenberger
 bindir: exe
 cert_chain: []
-date: 2025-03-07 00:00:00.000000000 Z
+date: 2025-03-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord