easy_ml 0.2.0.pre.rc41 → 0.2.0.pre.rc43

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 03333c45a1103bf7e75446a0c54d6799b62e64646abc1ab2abac123a206d1424
-  data.tar.gz: cb5ba985a5b8e5fd136b92e5ca5f65162d5189f06ba91bdd6e6763a69f5fbe56
+  metadata.gz: 6fc7f7afffd4e61312ab0302fb20e40a0eed3dcde05e16e9ea9ed1f523cb8681
+  data.tar.gz: d39511d86a3e01bc4ececcc20e1f7d0a583ac2eb07c8752c4ca2766b38318343
 SHA512:
-  metadata.gz: ddd88ca06fecf8366a7e3fa370b2f78e0e73ebc4c29fefcdd6cd0b208286710d503b767d6fabcf9af5eb40105bb0ffe63ddd773278b0cc9379750dfb4763d87f
-  data.tar.gz: 0c3b8dfdb6d293692439a1818dca5fe1974e27039895e54f08e492bb621c563a77cc4e0921f1df58603526a0b95354741684052aa9f1df8928e5b54de6f8caac
+  metadata.gz: 81f8594d04fcaa7274e7fc1a20ff8f67bd03599988f43e8377d10734639eb3aa48b8516f0422c7ca496774c2922e1e6042105cea72a7bad01c3bf847a553f636
+  data.tar.gz: d5c7cf29044fe2ac184b88ae2b68f1b9c12550157c331f263eb2e3ee79225c40245736c2531610141138e58753ffeaa257145052c02aa6fbe7598cafb0699a78

data/app/controllers/easy_ml/settings_controller.rb

@@ -29,11 +29,10 @@ module EasyML
       EasyML::Configuration.configure do |config|
         config.storage = @settings.storage
         config.timezone = @settings.timezone
-        config.s3_access_key_id = @settings.s3_access_key_id
-        config.s3_secret_access_key = @settings.s3_secret_access_key
         config.s3_bucket = @settings.s3_bucket
         config.s3_region = @settings.s3_region
         config.s3_prefix = @settings.s3_prefix
+        config.wandb_api_key = @settings.wandb_api_key
       end
       flash.now[:notice] = "Settings saved."
       render inertia: "pages/SettingsPage", props: {
@@ -47,8 +46,6 @@ module EasyML
       params.require(:settings).permit(
         :storage,
         :timezone,
-        :s3_access_key_id,
-        :s3_secret_access_key,
         :s3_bucket,
         :s3_region,
         :s3_prefix,

data/app/frontend/pages/SettingsPage.tsx

@@ -1,7 +1,7 @@
 import React, { useState } from 'react';
 import { usePage } from '@inertiajs/react'
 import { useInertiaForm } from 'use-inertia-form';
-import { Settings2, Save, AlertCircle, Key, Database, Globe2 } from 'lucide-react';
+import { Settings2, Save, AlertCircle, Key, Globe2, Database } from 'lucide-react';
 import { PluginSettings } from '../components/settings/PluginSettings';
 
 interface Settings {
@@ -9,9 +9,6 @@ interface Settings {
     timezone: string;
     s3_bucket: string;
     s3_region: string;
-    s3_access_key_id: string;
-    s3_secret_access_key: string;
-    wandb_api_key: string;
   }
 }
 
@@ -88,7 +85,6 @@ export default function SettingsPage({ settings: initialSettings }: { settings:
               <select
                 id="timezone"
                 value={formData.settings.timezone}
-                
                 onChange={(e) => setFormData({
                   ...formData,
                   settings: {
@@ -113,7 +109,6 @@ export default function SettingsPage({ settings: initialSettings }: { settings:
           {/* S3 Configuration */}
           <div className="space-y-4">
             <div className="flex items-center gap-2 mb-4">
-
               <Database className="w-5 h-5 text-gray-500" />
               <h3 className="text-lg font-medium text-gray-900">S3 Configuration</h3>
             </div>
@@ -162,80 +157,6 @@ export default function SettingsPage({ settings: initialSettings }: { settings:
                 </select>
               </div>
             </div>
-
-            <div className="bg-blue-50 rounded-lg p-4">
-              <div className="flex gap-2">
-                <AlertCircle className="w-5 h-5 text-blue-500 mt-0.5" />
-                <div>
-                  <h4 className="text-sm font-medium text-blue-900">AWS Credentials</h4>
-                  <p className="mt-1 text-sm text-blue-700">
-                    These credentials will be used as default for all S3 operations. You can override them per datasource.
-                  </p>
-                </div>
-              </div>
-            </div>
-
-            <div className="grid grid-cols-2 gap-6">
-              <div>
-                <label htmlFor="accessKeyId" className="block text-sm font-medium text-gray-700 mb-1">
-                  Access Key ID
-                </label>
-                <div className="relative">
-                  <Key className="absolute left-3 top-1/2 transform -translate-y-1/2 w-4 h-4 text-gray-400" />
-                  <input
-                    type="text"
-                    id="accessKeyId"
-                    value={formData.settings.s3_access_key_id}
-                    onChange={(e) => setFormData({
-                      ...formData,
-                      settings: {
-                        ...formData.settings,
-                        s3_access_key_id: e.target.value
-                      }
-                    })}
-                    className="mt-1 block w-full pl-9 rounded-md border-gray-300 shadow-sm focus:border-blue-500 focus:ring-blue-500"
-                    placeholder="AKIA..."
-                  />
-                </div>
-              </div>
-
-              <div>
-                <label htmlFor="secretAccessKey" className="block text-sm font-medium text-gray-700 mb-1">
-                  Secret Access Key
-                </label>
-                <div className="relative">
-                  <Key className="absolute left-3 top-1/2 transform -translate-y-1/2 w-4 h-4 text-gray-400" />
-                  <input
-                    type={showSecretKey ? 'text' : 'password'}
-                    id="secretAccessKey"
-                    value={formData.settings.s3_secret_access_key}
-                    onChange={(e) => setFormData({
-                      ...formData,
-                      settings: {
-                        ...formData.settings,
-                        s3_secret_access_key: e.target.value
-                      }
-                    })}
-                    className="mt-1 block w-full pl-9 pr-24 rounded-md border-gray-300 shadow-sm focus:border-blue-500 focus:ring-blue-500"
-                    placeholder="Your secret key"
-                  />
-                  <button
-                    type="button"
-                    onClick={() => setShowSecretKey(!showSecretKey)}
-                    className="absolute right-2 top-1/2 transform -translate-y-1/2 text-sm text-gray-500 hover:text-gray-700"
-                  >
-                    {showSecretKey ? 'Hide' : 'Show'}
-                  </button>
-                </div>
-              </div>
-            </div>
-          </div>
-
-          <div className="border-t border-gray-200 pt-8">
-            <PluginSettings
-              settings={formData.settings}
-              setData={(settings) => setFormData({ ...settings })}
-            />
           </div>
 
           <div className="pt-6 border-t flex items-center justify-between">

data/app/jobs/easy_ml/batch_job.rb

@@ -12,7 +12,14 @@ module EasyML
       # E.g. EasyML::ComputeFeatureBatchJob.enqueue_batch(features.map(&:id))
       #
       def enqueue_batch(args_list, batch_id = default_batch_id)
-        args_list = args_list.map { |arg| arg.is_a?(Array) ? arg : [arg] }
+        args_list = args_list.map do |arg|
+          arg = arg.is_a?(Array) ? arg : [arg]
+          arg.map do |arg|
+            arg.merge!(
+              batch_id: batch_id,
+            )
+          end
+        end
         store_batch_arguments(batch_id, args_list)
 
         args_list.each do |args|
@@ -22,8 +29,45 @@ module EasyML
         batch_id
       end
 
+      def enqueue_ordered_batches(args_list)
+        parent_id = get_parent_batch_id(args_list)
+        store_batch_arguments(parent_id, args_list)
+
+        batch = args_list.first
+        rest = args_list[1..]
+
+        rest.map do |batch|
+          Resque.redis.rpush("batch:#{parent_id}:remaining", batch.to_json)
+        end
+
+        enqueue_batch(batch)
+      end
+
+      def enqueue_next_batch(caller, parent_id)
+        next_batch = Resque.redis.lpop("batch:#{parent_id}:remaining")
+        payload = Resque.decode(next_batch)
+
+        caller.enqueue_batch(payload)
+      end
+
+      def next_batch?(parent_id)
+        batches_remaining(parent_id) > 0
+      end
+
+      def batches_remaining(parent_id)
+        Resque.redis.llen("batch:#{parent_id}:remaining")
+      end
+
+      def cleanup_batch(parent_id)
+        Resque.redis.del("batch:#{parent_id}:remaining")
+      end
+
       private
 
+      def get_parent_batch_id(args_list)
+        args_list.dup.flatten.first.dig(:parent_batch_id)
+      end
+
       # Store batch arguments in Redis
       def store_batch_arguments(batch_id, args_list)
         redis_key = "#{batch(batch_id)}:original_args"

data/app/jobs/easy_ml/compute_feature_job.rb

@@ -5,7 +5,6 @@ module EasyML
     @queue = :easy_ml
 
     def self.perform(batch_id, options = {})
-      puts "processing batch_id #{batch_id}"
       options.symbolize_keys!
       feature_id = options.dig(:feature_id)
       feature = EasyML::Feature.find(feature_id)
@@ -18,13 +17,12 @@ module EasyML
       end
 
       begin
+        feature.update(workflow_status: :analyzing) if feature.workflow_status == :ready
         feature.fit_batch(options.merge!(batch_id: batch_id))
       rescue => e
-        puts "Error computing feature: #{e.message}"
         EasyML::Feature.transaction do
           return if dataset.reload.workflow_status == :failed
 
-          puts "Logging error"
           feature.update(workflow_status: :failed)
           dataset.update(workflow_status: :failed)
           build_error_with_context(dataset, e, batch_id, feature)
@@ -42,12 +40,25 @@ module EasyML
 
     def self.after_batch_hook(batch_id, *args)
       puts "After batch!"
-      feature_ids = fetch_batch_arguments(batch_id).flatten.map(&:symbolize_keys).pluck(:feature_id).uniq
-      dataset = EasyML::Feature.find_by(id: feature_ids.first).dataset
-      dataset.after_fit_features
-    end
+      batch_args = fetch_batch_arguments(batch_id).flatten.map(&:symbolize_keys)
+      feature_ids = batch_args.pluck(:feature_id).uniq
+      parent_id = batch_args.pluck(:parent_batch_id).first
+
+      feature = EasyML::Feature.find_by(id: feature_ids.first)
+
+      if feature.failed?
+        dataset.features.where(workflow_status: :analyzing).update_all(workflow_status: :ready)
+        return BatchJob.cleanup_batch(parent_id)
+      end
+
+      feature.after_fit
 
-    def self.feature_fully_processed?(feature)
+      if BatchJob.next_batch?(parent_id)
+        BatchJob.enqueue_next_batch(self, parent_id)
+      else
+        dataset = EasyML::Feature.find_by(id: feature_ids.first).dataset
+        dataset.after_fit_features
+      end
     end
 
     private

data/app/models/easy_ml/feature.rb

@@ -196,12 +196,15 @@ module EasyML
         max_id = df[primary_key.last].max
       end
 
-      (min_id..max_id).step(batch_size).map do |batch_start|
+      (min_id..max_id).step(batch_size).map.with_index do |batch_start, idx|
         batch_end = [batch_start + batch_size, max_id + 1].min - 1
         {
           feature_id: id,
           batch_start: batch_start,
           batch_end: batch_end,
+          batch_number: feature_position,
+          subbatch_number: idx,
+          parent_batch_id: Random.uuid,
         }
       end
     end
@@ -211,17 +214,16 @@ module EasyML
     end
 
     def fit(features: [self], async: false)
-      # Sort features by position to ensure they're processed in order
-      features.update_all(workflow_status: :analyzing)
       ordered_features = features.sort_by(&:feature_position)
-      jobs = ordered_features.flat_map(&:build_batches)
+      jobs = ordered_features.map(&:build_batches)
 
       if async
-        EasyML::ComputeFeatureJob.enqueue_batch(jobs)
+        EasyML::ComputeFeatureJob.enqueue_ordered_batches(jobs)
       else
-        jobs.each do |job|
+        jobs.flatten.each do |job|
           EasyML::ComputeFeatureJob.perform(nil, job)
         end
+        features.each(&:after_fit) unless features.any?(&:failed?)
       end
     end
 
@@ -393,13 +395,11 @@ module EasyML
       updates = {
         applied_at: Time.current,
         needs_fit: false,
+        workflow_status: :ready,
       }.compact
       update!(updates)
     end
 
-    def fully_processed?
-    end
-
     private
 
     def bulk_update_positions(features)

data/app/models/easy_ml/model.rb

@@ -250,6 +250,7 @@ module EasyML
       bump_version(force: true)
       path = model_file.full_path(version)
       full_path = adapter.save_model_file(path)
+      puts "saving model to #{full_path}"
       model_file.upload(full_path)
 
       model_file.save
@@ -266,6 +267,7 @@ module EasyML
     end
 
     def cleanup
+      puts "keeping files #{files_to_keep}"
       get_model_file&.cleanup(files_to_keep)
     end
 
@@ -488,13 +490,9 @@ module EasyML
     end
 
     def root_dir
-      persisted = read_attribute(:root_dir)
+      relative_dir = read_attribute(:root_dir) || default_root_dir
 
-      if persisted.present? && !persisted.blank?
-        EasyML::Engine.root_dir.join(persisted).to_s
-      else
-        default_root_dir
-      end
+      EasyML::Engine.root_dir.join(relative_dir).to_s
     end
 
     def default_root_dir

data/app/models/easy_ml/model_file.rb

@@ -33,12 +33,23 @@ module EasyML
         s3_region: s3_region,
         s3_access_key_id: s3_access_key_id,
         s3_secret_access_key: s3_secret_access_key,
-        root_dir: full_dir,
+        root_dir: root_dir,
       )
     end
 
     def root_dir
-      model.root_dir
+      Pathname.new(model.root_dir)
+    end
+
+    def model_root
+      File.expand_path("..", root_dir.to_s)
+    end
+
+    def full_path(filename = nil)
+      filename = self.filename if filename.nil?
+      return nil if filename.nil?
+
+      root_dir.join(filename).to_s
     end
 
     def exist?
@@ -58,33 +69,7 @@ module EasyML
 
     def upload(path)
       synced_file.upload(path)
-      set_path(path)
-    end
-
-    def set_path(path)
-      path = get_full_path(path)
-      basename = Pathname.new(path).basename.to_s
-      unless path.start_with?(full_dir)
-        new_path = File.join(full_dir, basename).to_s
-        FileUtils.mkdir_p(Pathname.new(new_path).dirname.to_s)
-        FileUtils.cp(path, new_path)
-        path = new_path
-      end
-      self.filename = basename
-      self.path = get_relative_path(path)
-    end
-
-    def get_full_path(path)
-      path = path.to_s
-
-      path = Rails.root.join(path) unless path.match?(Regexp.new(Rails.root.to_s))
-      path
-    end
-
-    def get_relative_path(path)
-      path = path.to_s
-      path = path.to_s.split(Rails.root.to_s).last
-      path.to_s.split("/")[0..-2].reject(&:empty?).join("/")
+      update(filename: Pathname.new(path).basename.to_s)
     end
 
     def download
@@ -98,26 +83,6 @@ module EasyML
       Digest::SHA256.file(full_path).hexdigest
     end
 
-    def full_path(filename = nil)
-      filename = self.filename if filename.nil?
-      return nil if filename.nil?
-      return nil if relative_dir.nil?
-
-      Rails.root.join(relative_dir, filename).to_s
-    end
-
-    def relative_dir
-      root_dir.to_s.gsub(Regexp.new(Rails.root.to_s), "").gsub(%r{^/}, "")
-    end
-
-    def full_dir
-      Rails.root.join(relative_dir).to_s
-    end
-
-    def model_root
-      File.expand_path("..", full_dir)
-    end
-
     def cleanup!
       [model_root].each do |dir|
         EasyML::Support::FileRotate.new(dir, []).cleanup(extension_allowlist)

data/app/serializers/easy_ml/prediction_serializer.rb

@@ -5,7 +5,12 @@ module EasyML
     include JSONAPI::Serializer
 
     attribute :prediction do |object|
-      object.prediction_value.symbolize_keys.dig(:value)
+      case object.prediction_value
+      when Hash
+        object.prediction_value.symbolize_keys.dig(:value)
+      when Numeric
+        object.prediction_value
+      end
     end
 
     attributes :id,

data/lib/easy_ml/data/polars_reader.rb

@@ -225,7 +225,7 @@ module EasyML
 
         # Filter out any datetime columns, and use maybe_convert_date to convert later
         date_cols = (filtered[:dtypes] || {}).select { |k, v| v.class == Polars::Datetime }.keys
-        filtered[:dtypes] = (filtered[:dtypes] || {}).reject { |k, v| v.class == Polars::Datetime }
+        filtered[:dtypes] = (filtered[:dtypes] || {}).reject { |k, v| v.class == Polars::Datetime }.compact.to_h
         filtered = filtered.select { |k, _| supported_params.include?(k) }
         return filtered, date_cols
       end

data/lib/easy_ml/engine.rb

@@ -88,6 +88,17 @@ module EasyML
       end
     end
 
+    initializer "easy_ml.configure_secrets" do
+      EasyML::Configuration.configure do |config|
+        raise "S3_ACCESS_KEY_ID is missing. Set ENV['S3_ACCESS_KEY_ID']" unless ENV["S3_ACCESS_KEY_ID"]
+        raise "S3_SECRET_ACCESS_KEY is missing. Set ENV['S3_SECRET_ACCESS_KEY']" unless ENV["S3_SECRET_ACCESS_KEY"]
+
+        config.s3_access_key_id = ENV["S3_ACCESS_KEY_ID"]
+        config.s3_secret_access_key = ENV["S3_SECRET_ACCESS_KEY"]
+        config.wandb_api_key = ENV["WANDB_API_KEY"] if ENV["WANDB_API_KEY"]
+      end
+    end
+
     initializer "easy_ml.setup_generators" do |app|
       generators_path = EasyML::Engine.root.join("lib/easy_ml/railtie/generators")
       generators_dirs = Dir[File.join(generators_path, "**", "*.rb")]

data/lib/easy_ml/predict.rb

@@ -10,23 +10,38 @@ module EasyML
       @models = {}
     end
 
-    def self.predict(model_name, df)
+    def self.predict(model_name, df, serialize: false)
       if df.is_a?(Hash)
         df = Polars::DataFrame.new(df)
       end
-      raw_input = df.to_hashes&.first
+      raw_input = df.to_hashes
       df = instance.normalize(model_name, df)
+      normalized_input = df.to_hashes
       preds = instance.predict(model_name, df)
       current_version = instance.get_model(model_name)
 
-      EasyML::Prediction.create!(
-        model: current_version.model,
-        model_history: current_version,
-        prediction_type: current_version.model.task,
-        prediction_value: preds.first,
-        raw_input: raw_input,
-        normalized_input: df.to_hashes&.first,
-      )
+      output = preds.zip(raw_input, normalized_input).map do |pred, raw, norm|
+        EasyML::Prediction.create!(
+          model: current_version.model,
+          model_history: current_version,
+          prediction_type: current_version.model.task,
+          prediction_value: pred,
+          raw_input: raw,
+          normalized_input: norm,
+        )
+      end
+
+      output = if output.is_a?(Array) && output.count == 1
+          output.first
+        else
+          output
+        end
+
+      if serialize
+        EasyML::PredictionSerializer.new(output).serializable_hash
+      else
+        output
+      end
     end
 
     def self.train(model_name, tuner: nil, evaluator: nil)

data/lib/easy_ml/railtie/generators/migration/migration_generator.rb

@@ -39,6 +39,7 @@ module EasyML
             create_easy_ml_predictions
             create_easy_ml_event_contexts
             add_workflow_status_to_easy_ml_features
+            drop_path_from_easy_ml_model_files
           ].freeze
 
           # Specify the next migration number

data/lib/easy_ml/railtie/templates/migration/add_workflow_status_to_easy_ml_features.rb.tt

@@ -4,5 +4,10 @@ class AddWorkflowStatusToEasyMLFeatures < ActiveRecord::Migration[<%= ActiveReco
       add_column :easy_ml_features, :workflow_status, :string
       add_index :easy_ml_features, :workflow_status
     end
+
+    unless column_exists?(:easy_ml_feature_histories, :workflow_status)
+      add_column :easy_ml_feature_histories, :workflow_status, :string
+      add_index :easy_ml_feature_histories, :workflow_status
+    end
   end
 end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_retraining_jobs.rb.tt

@@ -32,6 +32,7 @@ class CreateEasyMLRetrainingJobs < ActiveRecord::Migration[<%= ActiveRecord::Mig
         t.index :auto_deploy
         t.index :tuning_enabled
       end
+    end
 
     unless table_exists?(:easy_ml_retraining_runs)
       create_table :easy_ml_retraining_runs do |t|

data/lib/easy_ml/railtie/templates/migration/drop_path_from_easy_ml_model_files.rb.tt

@@ -0,0 +1,11 @@
+class DropPathFromEasyMLModelFiles < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    if column_exists?(:easy_ml_model_files, :path)
+      remove_column :easy_ml_model_files, :path
+    end
+
+    if column_exists?(:easy_ml_model_file_histories, :path)
+      remove_column :easy_ml_model_file_histories, :path
+    end
+  end
+end

data/lib/easy_ml/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module EasyML
-  VERSION = "0.2.0-rc41"
+  VERSION = "0.2.0-rc43"
 
   module Version
   end

data/public/easy_ml/assets/.vite/manifest.json

@@ -1,11 +1,11 @@
 {
   "entrypoints/Application.tsx": {
-    "file": "assets/entrypoints/Application.tsx-DF5SSkYi.js",
+    "file": "assets/entrypoints/Application.tsx-jPsqOyb0.js",
     "name": "entrypoints/Application.tsx",
     "src": "entrypoints/Application.tsx",
     "isEntry": true,
     "css": [
-      "assets/Application-Cu7lNJmG.css"
+      "assets/Application-zpGA_Q9c.css"
     ]
   }
 }