easy_ml 0.2.0.pre.rc77 → 0.2.0.pre.rc81

data/lib/easy_ml/core/tuner.rb

@@ -8,7 +8,7 @@ module EasyML
                     :metrics, :objective, :n_trials, :direction, :evaluator,
                     :study, :results, :adapter, :tune_started_at, :x_valid, :y_valid,
                     :project_name, :job, :current_run, :trial_enumerator, :progress_block,
-                    :tuner_job, :dataset
+                    :tuner_job, :dataset, :x_normalized
 
       def initialize(options = {})
         @model = options[:model]
@@ -73,9 +73,12 @@ module EasyML
         model.task = task
 
         model.dataset.refresh if model.dataset.needs_refresh?
-        x_valid, y_valid = model.dataset.valid(split_ys: true, select: model.dataset.col_order)
+        x_valid, y_valid = model.dataset.valid(split_ys: true, all_columns: true)
+        x_normalized = model.dataset.normalize(x_valid, inference: true)
+        x_normalized = model.preprocess(x_normalized)
         self.x_valid = x_valid
         self.y_valid = y_valid
+        self.x_normalized = x_normalized
         self.dataset = model.dataset.valid(all_columns: true)
         adapter.tune_started_at = tune_started_at
         adapter.x_valid = x_valid
@@ -99,13 +102,12 @@ module EasyML
             @study.tell(@current_trial, result)
           rescue StandardError => e
             puts EasyML::Event.easy_ml_context(e.backtrace)
-            @tuner_run.update!(status: :failed, hyperparameters: {})
+            @tuner_run.update!(status: :failed, hyperparameters: model.hyperparameters.to_h)
             puts "Optuna failed with: #{e.message}"
             raise e
           end
         end
 
-        model.after_tuning
         return nil if tuner_job.tuner_runs.all?(&:failed?)
 
         best_run = tuner_job.best_run
@@ -115,6 +117,13 @@ module EasyML
           status: :success,
           completed_at: Time.current,
         )
+        model.after_tuning
+        if best_run&.hyperparameters.present?
+          model.hyperparameters = best_run.hyperparameters
+          model.fit
+          model.save
+        end
+        model.cleanup
 
         best_run&.hyperparameters
       rescue StandardError => e
@@ -138,7 +147,7 @@ module EasyML
           end
         end
 
-        y_pred = model.predict(x_valid)
+        y_pred = model.predict(x_normalized)
         model.metrics = metrics
         metrics = model.evaluate(y_pred: y_pred, y_true: y_valid, x_true: x_valid, dataset: dataset)
         metric = metrics.symbolize_keys.dig(model.evaluator[:metric].to_sym)

data/lib/easy_ml/data/dataset_manager/reader/base.rb

@@ -35,6 +35,18 @@ module EasyML
 
           private
 
+          def list_df_nulls(df)
+            df = df.lazy
+
+            columns = df.columns
+            selects = columns.map do |col|
+              Polars.col(col).null_count.alias(col)
+            end
+            null_info = df.select(selects).collect
+            null_info.to_hashes.first.compact
+            null_info.to_hashes.first.transform_values { |v| v > 0 ? v : nil }.compact.keys
+          end
+
           def apply_defaults(kwargs)
             options = kwargs.dup
 

data/lib/easy_ml/data/dataset_manager/reader/data_frame.rb

@@ -1,4 +1,3 @@
-
 module EasyML
   module Data
     class DatasetManager
@@ -8,11 +7,17 @@ module EasyML
             return query_dataframes(lazy_frames, schema)
           end
 
+          def list_nulls
+            df = lazy_frames
+            list_df_nulls(df)
+          end
+
           def schema
             input.schema
           end
 
-        private
+          private
+
           def lazy_frames
             input.lazy
           end
@@ -20,4 +25,4 @@ module EasyML
       end
     end
   end
-end
+end

data/lib/easy_ml/data/dataset_manager/reader/file.rb

@@ -15,6 +15,11 @@ module EasyML
             return Batch.new(options, &block).query
           end
 
+          def list_nulls
+            df = dataframe.lazy
+            list_df_nulls(df)
+          end
+
           def schema
             @schema ||= files.any? ? Polars.read_parquet_schema(files.first) : nil
           end

data/lib/easy_ml/data/dataset_manager/reader.rb

@@ -17,12 +17,18 @@ module EasyML
           ).query
         end
 
-        def self.schema(input, **kwargs, &block)
+        def self.schema(input = nil, **kwargs, &block)
           adapter(input).new(
             kwargs.merge!(input: input), &block
           ).schema
         end
 
+        def self.list_nulls(input = nil, **kwargs, &block)
+          adapter(input).new(
+            kwargs.merge!(input: input), &block
+          ).list_nulls
+        end
+
         def self.files(dir)
           Dir.glob(::File.join(dir, "**/*.{parquet}"))
         end

data/lib/easy_ml/data/dataset_manager/writer/base.rb

@@ -36,6 +36,10 @@ module EasyML
             clear_unique_id
           end
 
+          def unlock!
+            clear_all_keys
+          end
+
           private
 
           def files
@@ -50,6 +54,10 @@ module EasyML
             safe_write(df, unique_path(subdir: subdir))
           end
 
+          def acquire_lock(key, &block)
+            Support::Lockable.with_lock("#{key}:lock", wait_timeout: 2, &block)
+          end
+
           def unique_path(subdir: nil)
             filename = [filenames, unique_id(subdir: subdir), "parquet"].compact.join(".")
 
@@ -63,15 +71,18 @@ module EasyML
           end
 
           def clear_all_keys
-            keys = list_keys
-            Support::Lockable.with_lock(keys, wait_timeout: 2) do |suo|
-              suo.client.del(keys)
+            list_keys.each { |key| unlock_file(key) }
+          end
+
+          def unlock_file(key)
+            acquire_lock(key) do |suo|
+              suo.client.del(key)
             end
           end
 
           def clear_unique_id(subdir: nil)
             key = unique_id_key(subdir: subdir)
-            Support::Lockable.with_lock(key, wait_timeout: 2) do |suo|
+            acquire_lock(key) do |suo|
               suo.client.del(key)
             end
           end
@@ -83,7 +94,7 @@ module EasyML
           def add_key(key)
             keylist = unique_id_key(subdir: "keylist")
 
-            Support::Lockable.with_lock(keylist, wait_timeout: 2) do |suo|
+            acquire_lock(keylist) do |suo|
               suo.client.sadd(keylist, key)
             end
           end
@@ -91,14 +102,20 @@ module EasyML
           def list_keys
             keylist = unique_id_key(subdir: "keylist")
 
-            Support::Lockable.with_lock(keylist, wait_timeout: 2) do |suo|
-              suo.client.smembers(keylist)
+            acquire_lock(keylist) do |suo|
+              if suo.client.type(keylist) == "set"
+                suo.client.smembers(keylist)
+              else
+                suo.client.del(keylist)
+                []
+              end
             end
           end
 
           def key_exists?(key)
             keylist = unique_id_key(subdir: "keylist")
-            Support::Lockable.with_lock(keylist, wait_timeout: 2) do |suo|
+
+            acquire_lock(keylist) do |suo|
               suo.client.sismember(keylist, key)
             end
           end
@@ -107,7 +124,7 @@ module EasyML
             key = unique_id_key(subdir: subdir)
             add_key(key)
 
-            Support::Lockable.with_lock(key, wait_timeout: 2) do |suo|
+            acquire_lock(key) do |suo|
               redis = suo.client
 
               seq = (redis.get(key) || "0").to_i

data/lib/easy_ml/data/dataset_manager/writer.rb

@@ -15,7 +15,7 @@ module EasyML
         ]
 
         attr_accessor :filenames, :root_dir, :partition,
-                      :append_only, :primary_key, :options
+                      :primary_key, :options, :append_only, :named
 
         def initialize(options)
           @root_dir = options.dig(:root_dir)
@@ -27,6 +27,10 @@ module EasyML
           @options = options
         end
 
+        def unlock!
+          adapter_class.new(options).unlock!
+        end
+
         def store(df, *args)
           adapter_class.new(options.merge!(df: df)).store(*args)
         end

data/lib/easy_ml/data/dataset_manager.rb

@@ -44,13 +44,21 @@ module EasyML
           Reader.schema(input, **kwargs, &block)
         end
 
+        def list_nulls(input = nil, **kwargs, &block)
+          Reader.list_nulls(input, **kwargs, &block)
+        end
+
         def num_rows
           Reader.num_rows
         end
       end
 
-      def num_rows
-        Reader.num_rows(root_dir)
+      def list_nulls(input = nil, **kwargs, &block)
+        Reader.list_nulls(input, **kwargs, &block)
+      end
+
+      def num_rows(input = nil, **kwargs, &block)
+        Reader.num_rows(input, **kwargs, &block)
       end
 
       def query(input = nil, **kwargs, &block)
@@ -67,6 +75,8 @@ module EasyML
         Reader.sha(root_dir)
       end
 
+      # Transform CSV files into Parquet files, of all the same datatype.
+      # Learn datatypes of columns and store schema.
       def normalize
         Normalizer.normalize(root_dir)
       end
@@ -75,14 +85,18 @@ module EasyML
         query
       end
 
-      def store(df, *args)
-        writer.store(df, *args)
+      def unlock!
+        writer.unlock!
       end
 
       def compact
         writer.compact
       end
 
+      def store(df, *args)
+        writer.store(df, *args)
+      end
+
       def cp(from, to)
         writer.cp(from, to)
       end

data/lib/easy_ml/data/embeddings/adapters.rb

@@ -0,0 +1,56 @@
+module EasyML
+  module Data
+    class Embeddings
+      class Adapters
+        attr_accessor :model, :config
+
+        ADAPTERS = {
+          anthropic: Langchain::LLM::Anthropic,
+          gemini: Langchain::LLM::GoogleGemini,
+          openai: Langchain::LLM::OpenAI,
+          ollama: Langchain::LLM::Ollama,
+        }
+
+        DEFAULTS = {
+          api_key: {
+            anthropic: ENV["ANTHROPIC_API_KEY"],
+            gemini: ENV["GEMINI_API_KEY"],
+            openai: ENV["OPENAI_API_KEY"],
+            ollama: ENV["OLLAMA_API_KEY"],
+          },
+        }
+
+        def initialize(model, config = {})
+          @model = model.to_sym
+          @config = config.symbolize_keys
+          apply_defaults
+        end
+
+        def embed(df, col)
+          pick
+          texts = df[col].to_a
+          df = df.with_column(
+            embeddings: adapter.embed(text: texts),
+          )
+        end
+
+        private
+
+        def pick
+          @adapter ||= ADAPTERS[@model].new(config)
+          self
+        end
+
+        def apply_defaults
+          @config = @config.deep_symbolize_keys
+
+          DEFAULTS.each do |k, v|
+            unless @config.key?(k)
+              @config[k] = v[@model]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/easy_ml/data/embeddings.rb

@@ -0,0 +1,43 @@
+module EasyML
+  module Data
+    class Embeddings
+      COMPRESSION_DEFAULT = {
+        present: :balanced,
+      }
+
+      attr_reader :df, :column, :model, :adapter, :compression,
+                  :embeddings, :compressed_embeddings
+
+      def initialize(options = {})
+        @df = options[:df]
+        @column = options[:column]
+        @model = options[:model]
+        @config = options[:config] || {}
+        @compression = options[:compression] || COMPRESSION_DEFAULT
+      end
+
+      def create
+        embed
+        compress
+      end
+
+      def embed
+        @embeddings ||= adapter.embed(df, column)
+      end
+
+      def compress
+        @compressed_embeddings ||= compression_adapter.compress(embeddings)
+      end
+
+      private
+
+      def adapter
+        @adapter ||= EasyML::Data::Embeddings::Adapters.new(model, config)
+      end
+
+      def compression_adapter
+        @compression_adapter ||= EasyML::Data::Embeddings::Compression.new(compression)
+      end
+    end
+  end
+end

data/lib/easy_ml/data/polars_column.rb

@@ -124,11 +124,25 @@ module EasyML
       # @param series [Polars::Series] The string series to analyze
       # @return [Symbol] One of :datetime, :text, or :categorical
       def determine_string_type(series)
-        if EasyML::Data::DateConverter.maybe_convert_date(Polars::DataFrame.new({ temp: series }),
-                                                          :temp)[:temp].dtype.is_a?(Polars::Datetime)
-          :datetime
-        else
-          categorical_or_text?(series)
+        # Try to parse as numeric first
+        begin
+          # Try integer first
+          series.cast(Polars::Int64)
+          return :numeric
+        rescue StandardError
+          begin
+            # Try float if integer fails
+            series.cast(Polars::Float64)
+            return :numeric
+          rescue StandardError
+            # If not numeric, check for datetime or categorical
+            if EasyML::Data::DateConverter.maybe_convert_date(Polars::DataFrame.new({ temp: series }),
+                                                              :temp)[:temp].dtype.is_a?(Polars::Datetime)
+              :datetime
+            else
+              categorical_or_text?(series)
+            end
+          end
         end
       end
 

data/lib/easy_ml/engine.rb

@@ -55,7 +55,7 @@ module EasyML
       Polars.enable_string_cache
     end
 
-    if %w[db:migrate db:migrate:status db:setup db:drop assets:precompile].include?(ARGV.first)
+    if %w[db:create db:migrate db:migrate:status db:setup db:drop assets:precompile].include?(ARGV.first)
       config.eager_load_paths = config.eager_load_paths.without(config.eager_load_paths.map(&:to_s).grep(/easy_ml/).map { |p| Pathname.new(p) })
     else
       config.after_initialize do
@@ -77,6 +77,21 @@ module EasyML
       end
     end
 
+    unless %w[db:create db:migrate db:migrate:status db:setup db:drop assets:precompile].include?(ARGV.first)
+      initializer "easy_ml.configure_secrets" do
+        EasyML::Configuration.configure do |config|
+          raise "S3_ACCESS_KEY_ID is missing. Set ENV['S3_ACCESS_KEY_ID']" unless ENV["S3_ACCESS_KEY_ID"]
+          raise "S3_SECRET_ACCESS_KEY is missing. Set ENV['S3_SECRET_ACCESS_KEY']" unless ENV["S3_SECRET_ACCESS_KEY"]
+
+          config.s3_access_key_id = ENV["S3_ACCESS_KEY_ID"]
+          config.s3_secret_access_key = ENV["S3_SECRET_ACCESS_KEY"]
+          config.s3_region = ENV["S3_REGION"] ? ENV["S3_REGION"] : "us-east-1"
+          config.timezone = ENV["TIMEZONE"].present? ? ENV["TIMEZONE"] : "America/New_York"
+          config.wandb_api_key = ENV["WANDB_API_KEY"] if ENV["WANDB_API_KEY"]
+        end
+      end
+    end
+
     initializer "easy_ml.check_pending_migrations" do
       if defined?(Rails::Server)
         config.after_initialize do
@@ -96,19 +111,6 @@ module EasyML
       end
     end
 
-    initializer "easy_ml.configure_secrets" do
-      EasyML::Configuration.configure do |config|
-        raise "S3_ACCESS_KEY_ID is missing. Set ENV['S3_ACCESS_KEY_ID']" unless ENV["S3_ACCESS_KEY_ID"]
-        raise "S3_SECRET_ACCESS_KEY is missing. Set ENV['S3_SECRET_ACCESS_KEY']" unless ENV["S3_SECRET_ACCESS_KEY"]
-
-        config.s3_access_key_id = ENV["S3_ACCESS_KEY_ID"]
-        config.s3_secret_access_key = ENV["S3_SECRET_ACCESS_KEY"]
-        config.s3_region = ENV["S3_REGION"] if ENV["S3_REGION"]
-        config.timezone = ENV["TIMEZONE"].present? ? ENV["TIMEZONE"] : "America/New_York"
-        config.wandb_api_key = ENV["WANDB_API_KEY"] if ENV["WANDB_API_KEY"]
-      end
-    end
-
     initializer "easy_ml.setup_generators" do |app|
       generators_path = EasyML::Engine.root.join("lib/easy_ml/railtie/generators")
       generators_dirs = Dir[File.join(generators_path, "**", "*.rb")]

data/lib/easy_ml/feature_store.rb

@@ -5,19 +5,22 @@ module EasyML
     def initialize(feature)
       @feature = feature
 
-      datasource_config = feature.dataset.datasource.configuration || {}
-
-      options = {
-        root_dir: feature_dir,
-        filenames: "feature",
-        append_only: false,
-        primary_key: feature.primary_key&.first,
-        partition_size: batch_size,
-        s3_bucket: datasource_config.dig("s3_bucket") || EasyML::Configuration.s3_bucket,
-        s3_prefix: s3_prefix,
-        polars_args: datasource_config.dig("polars_args"),
-      }.compact
-      super(options)
+      datasource_config = feature&.dataset&.datasource&.configuration
+      if datasource_config
+        options = {
+          root_dir: feature_dir,
+          filenames: "feature",
+          append_only: false,
+          primary_key: feature.primary_key&.first,
+          partition_size: batch_size,
+          s3_bucket: datasource_config.dig("s3_bucket") || EasyML::Configuration.s3_bucket,
+          s3_prefix: s3_prefix,
+          polars_args: datasource_config.dig("polars_args"),
+        }.compact
+        super(options)
+      else
+        super({ root_dir: "" })
+      end
     end
 
     def cp(old_version, new_version)
@@ -30,7 +33,7 @@ module EasyML
       files_to_cp = Dir.glob(Pathname.new(old_dir).join("**/*")).select { |f| File.file?(f) }
 
       files_to_cp.each do |file|
-        target_file = file.gsub(old_version.to_s, new_version.to_s)
+        target_file = file.gsub(old_dir, new_dir)
         FileUtils.mkdir_p(File.dirname(target_file))
         FileUtils.cp(file, target_file)
       end
@@ -46,9 +49,9 @@ module EasyML
       File.join(
         Rails.root,
         "easy_ml/datasets",
-        feature.dataset.name.parameterize.gsub("-", "_"),
+        feature&.dataset&.name&.parameterize&.gsub("-", "_"),
         "features",
-        feature.name.parameterize.gsub("-", "_"),
+        feature&.name&.parameterize&.gsub("-", "_"),
         version.to_s
       )
     end

data/lib/easy_ml/support/lockable.rb

@@ -22,7 +22,7 @@ module EasyML
       end
 
       def self.lock_client(key, wait_timeout: 0.1, stale_timeout: 60 * 10, resources: 1)
-        Suo::Client::Redis.new(prefixed_key(key), {
+        Suo::Client::Redis.new(key, {
           acquisition_timeout: wait_timeout,
           stale_lock_expiry: stale_timeout,
           resources: resources,
@@ -30,10 +30,6 @@ module EasyML
         })
       end
 
-      def self.prefixed_key(key)
-        "easy_ml:#{key}"
-      end
-
       # Execute a block with a Redis lock
       def self.with_lock(key, wait_timeout: 0.1, stale_timeout: 60 * 10, resources: 1)
         lock_key = nil

data/lib/easy_ml/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module EasyML
-  VERSION = "0.2.0-rc77"
+  VERSION = "0.2.0-rc81"
 
   module Version
   end

data/public/easy_ml/assets/.vite/manifest.json

@@ -1,6 +1,6 @@
 {
   "entrypoints/Application.tsx": {
-    "file": "assets/entrypoints/Application.tsx-B1qLZuyu.js",
+    "file": "assets/entrypoints/Application.tsx-Bbf3mD_b.js",
     "name": "entrypoints/Application.tsx",
     "src": "entrypoints/Application.tsx",
     "isEntry": true,