easy_ml 0.2.0.pre.rc57 → 0.2.0.pre.rc60

data/app/models/easy_ml/feature.rb

@@ -55,6 +55,7 @@ module EasyML
     end
 
     belongs_to :dataset, class_name: "EasyML::Dataset"
+    has_many :columns, class_name: "EasyML::Column", dependent: :destroy
 
     validates :feature_class, presence: true
     validates :feature_position, presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 0 }
@@ -72,7 +73,7 @@ module EasyML
       end
 
       # Combine all conditions with OR
-      where(id: where(needs_fit: true).or(where(conditions.join(" OR "))).select { |f| f.adapter.respond_to?(:fit) }.map(&:id))
+      where(id: where(needs_fit: true).or(where(conditions.join(" OR "))).map(&:id))
     }
     scope :never_applied, -> { where(applied_at: nil) }
     scope :never_fit, -> do
@@ -81,6 +82,7 @@ module EasyML
             where(id: fittable.map(&:id))
           end
     scope :needs_fit, -> { has_changes.or(never_applied).or(never_fit) }
+    scope :ready_to_apply, -> { where.not(id: needs_fit.map(&:id)) }
 
     before_save :apply_defaults, if: :new_record?
     before_save :update_sha
@@ -165,6 +167,13 @@ module EasyML
       end
     end
 
+    def computes_columns
+      unless adapter.respond_to?(:computes_columns)
+        raise "Feature #{feature_class} must declare which columns it computes using the :computes_columns method"
+      end
+      adapter.computes_columns
+    end
+
     def build_batches
       if batchable?
         batch
@@ -216,8 +225,11 @@ module EasyML
     def fit(features: [self], async: false)
       ordered_features = features.sort_by(&:feature_position)
       jobs = ordered_features.map(&:build_batches)
+      job_count = jobs.dup.flatten.size
 
-      if async
+      # This is very important! For whatever reason, Resque BatchJob does not properly
+      # handle batch finished callbacks for batch size = 1
+      if async && job_count > 1
         EasyML::ComputeFeatureJob.enqueue_ordered_batches(jobs)
       else
         jobs.flatten.each do |job|
@@ -233,13 +245,14 @@ module EasyML
       if batch_args.key?(:batch_start)
         actually_fit_batch(batch_args)
       else
-        actually_fit_batch(get_batch_args(**batch_args))
+        batch_args = get_batch_args(**batch_args)
+        actually_fit_batch(batch_args)
       end
     end
 
     # Transform a single batch, used for testing the user's feature implementation
     def transform_batch(df = nil, batch_args = {})
-      if df.present?
+      if df.is_a?(Polars::DataFrame)
         actually_transform_batch(df)
       else
         actually_transform_batch(build_batch(get_batch_args(**batch_args)))
@@ -281,12 +294,14 @@ module EasyML
         batch_args.symbolize_keys!
 
         if adapter.respond_to?(:batch)
-          batch_df = adapter.fit(dataset.raw, self, batch_args)
+          df = dataset.raw
         else
           df = build_batch(batch_args)
-          batch_df = adapter.fit(df, self, batch_args)
         end
       end
+      return if df.blank?
+
+      batch_df = adapter.fit(df, self, batch_args)
       if batch_df.present?
         store(batch_df)
       else
@@ -296,10 +311,14 @@ module EasyML
     end
 
     def actually_transform_batch(df)
-      return nil unless df.present?
-      return df if adapter.respond_to?(:fit) && feature_store.empty?
+      return nil unless df.is_a?(Polars::DataFrame)
+      return df if !adapter.respond_to?(:transform) && feature_store.empty?
 
+      df_len_was = df.shape[0]
       result = adapter.transform(df, self)
+      raise "Feature '#{name}' must return a Polars::DataFrame, got #{result.class}" unless result.is_a?(Polars::DataFrame)
+      df_len_now = result.shape[0]
+      raise "Feature #{feature_class}#transform: output size must match input size! Input size: #{df_len_now}, output size: #{df_len_was}." if df_len_now != df_len_was
       update!(applied_at: Time.current)
       result
     end
@@ -377,8 +396,8 @@ module EasyML
       feature_store.list_partitions
     end
 
-    def query(filter: nil)
-      feature_store.query(filter: filter)
+    def query(**kwargs)
+      feature_store.query(**kwargs)
     end
 
     def store(df)

data/app/models/easy_ml/feature_history.rb

@@ -31,6 +31,18 @@ module EasyML
 
     after_find :download_remote_files
     scope :ordered, -> { order(feature_position: :asc) }
+    scope :ready_to_apply, -> { where.not(id: needs_fit.map(&:id)) }
+    scope :has_changes, lambda {
+      none
+    }
+    scope :never_applied, -> { where(applied_at: nil) }
+    scope :never_fit, -> do
+            fittable = where(fit_at: nil)
+            fittable = fittable.select { |f| f.adapter.respond_to?(:fit) }
+            where(id: fittable.map(&:id))
+          end
+    scope :needs_fit, -> { has_changes.or(never_applied).or(never_fit) }
+    scope :ready_to_apply, -> { where.not(id: needs_fit.map(&:id)) }
 
     def download_remote_files
       feature_store&.download

data/app/models/easy_ml/feature_list.rb

@@ -0,0 +1,15 @@
+module EasyML
+  module FeatureList
+    def feature_list
+      self
+    end
+
+    def dataset
+      proxy_association.owner
+    end
+
+    def computed_column_names
+      flat_map(&:computes_columns).uniq
+    end
+  end
+end

data/app/models/easy_ml/model.rb

@@ -17,6 +17,7 @@
 #  is_training     :boolean
 #  created_at      :datetime         not null
 #  updated_at      :datetime         not null
+#  slug            :string           not null
 #
 require_relative "models/hyperparameters"
 
@@ -66,6 +67,7 @@ module EasyML
     after_initialize :bump_version, if: -> { new_record? }
     after_initialize :set_defaults, if: -> { new_record? }
     before_save :save_model_file, if: -> { is_fit? && !is_history_class? && model_changed? && !@skip_save_model_file }
+    before_validation :set_slug, if: :name_changed?
 
     VALID_TASKS = %i[regression classification].freeze
 
@@ -91,6 +93,7 @@ module EasyML
                      }
     validates :model_type, inclusion: { in: MODEL_NAMES }
     validates :dataset_id, presence: true
+    validates :slug, presence: true, uniqueness: true
     validate :validate_metrics_allowed
     before_save :set_root_dir
 
@@ -189,6 +192,7 @@ module EasyML
         evaluator: evaluator,
         model: self,
         dataset: dataset,
+        metrics: metrics,
       }.compact
       tuner.merge!(extra_params)
       tuner_instance = EasyML::Core::Tuner.new(tuner)
@@ -307,7 +311,6 @@ module EasyML
 
       dataset.refresh
       adapter.fit(tuning: tuning, x_train: x_train, y_train: y_train, x_valid: x_valid, y_valid: y_valid, &progress_block)
-      @is_fit = true
     end
 
     def batch_args
@@ -334,11 +337,8 @@ module EasyML
 
     def fit_in_batches(tuning: false, batch_size: nil, batch_overlap: nil, batch_key: nil, checkpoint_dir: Rails.root.join("tmp", "xgboost_checkpoints"), &progress_block)
       adapter.fit_in_batches(tuning: tuning, batch_size: batch_size, batch_overlap: batch_overlap, batch_key: batch_key, checkpoint_dir: checkpoint_dir, &progress_block)
-      @is_fit = true
     end
 
-    attr_accessor :is_fit
-
     def is_fit?
       model_file = get_model_file
       return true if model_file.present? && model_file.fit?
@@ -447,6 +447,21 @@ module EasyML
       )
     end
 
+    include Rails.application.routes.mounted_helpers
+
+    def api_fields
+      {
+        url: EasyML::Engine.routes.url_helpers.predictions_path,
+        method: "POST",
+        data: {
+          model: slug,
+          input: dataset.columns.api_inputs.sort_by_required.map(&:to_api).each_with_object({}) do |field, hash|
+            hash[field[:name]] = field.except(:name)
+          end,
+        },
+      }
+    end
+
     class CannotdeployError < StandardError
     end
 
@@ -606,6 +621,12 @@ module EasyML
       errors.add(:metrics,
                  "don't know how to handle #{"metrics".pluralize(unknown_metrics)} #{unknown_metrics.join(", ")}, use EasyML::Core::ModelEvaluator.register(:name, Evaluator, :regression|:classification)")
     end
+
+    def set_slug
+      if slug.nil? && name.present?
+        self.slug = name.gsub(/\s/, "_").downcase
+      end
+    end
   end
 end
 

data/app/models/easy_ml/model_history.rb

@@ -22,6 +22,7 @@
 #  history_ended_at   :datetime
 #  history_user_id    :integer
 #  snapshot_id        :string
+#  slug               :string
 #
 module EasyML
   class ModelHistory < ActiveRecord::Base

data/app/models/easy_ml/retraining_run.rb

@@ -158,6 +158,7 @@ module EasyML
         model: training_model,
         y_pred: y_pred,
         y_true: y_true,
+        dataset: training_model.dataset.test(all_columns: true),
         evaluator: evaluator,
       )
       metric_value = metrics[metric]

data/app/serializers/easy_ml/column_serializer.rb

@@ -19,9 +19,19 @@
 #
 module EasyML
   class ColumnSerializer
+    class SmallSerializer
+      include JSONAPI::Serializer
+      attributes :id, :name
+    end
+
     include JSONAPI::Serializer
 
     attributes :id, :name, :description, :dataset_id, :datatype, :polars_datatype, :preprocessing_steps,
-               :hidden, :drop_if_null, :sample_values, :statistics, :is_target
+               :hidden, :drop_if_null, :sample_values, :statistics, :is_target,
+               :is_computed, :computed_by, :lineage
+
+    attribute :required do |object|
+      object.required?
+    end
   end
 end

data/app/serializers/easy_ml/dataset_serializer.rb

@@ -24,6 +24,27 @@ require_relative "./column_serializer"
 #
 module EasyML
   class DatasetSerializer
+    class SmallSerializer
+      include JSONAPI::Serializer
+
+      attributes :id, :name, :description, :target, :num_rows, :status,
+                 :datasource_id, :preprocessing_steps, :workflow_status, :statistics
+
+      attribute :columns do |dataset|
+        dataset.columns.order(:id).map do |column|
+          ColumnSerializer::SmallSerializer.new(column).serializable_hash.dig(:data, :attributes)
+        end
+      end
+      attribute :stacktrace do |object|
+        if !object.failed? || object.events.empty?
+          nil
+        else
+          last_event = object.events.where(status: :failed).order(id: :desc).limit(1).last
+          last_event&.stacktrace
+        end
+      end
+    end
+
     include JSONAPI::Serializer
 
     attributes :id, :name, :description, :target, :num_rows, :status,
@@ -47,7 +68,7 @@ module EasyML
       if dataset.workflow_status.to_sym == :analyzing
         nil
       else
-        dataset.data(limit: 10, all_columns: true)&.to_hashes
+        dataset.data(limit: 10, all_columns: true, refresh: false)&.to_hashes || dataset.raw.data(limit: 10, all_columns: true).to_hashes
       end
     end
 
@@ -62,7 +83,7 @@ module EasyML
     end
 
     attribute :needs_refresh do |dataset|
-      dataset.needs_refresh?
+      dataset.needs_refresh?(exclude: [:datasource_needs_refresh])
     end
 
     attribute :stacktrace do |object|

data/config/initializers/enumerable.rb

@@ -0,0 +1,17 @@
+module Enumerable
+  def count_by(&block)
+    self.group_by(&block).inject({}) do |h, (k, v)|
+      h.tap do
+        h[k] = v.count
+      end
+    end
+  end
+
+  def key_by(&block)
+    self.group_by(&block).inject({}) do |h, (k, v)|
+      h.tap do
+        h[k] = v.first
+      end
+    end
+  end
+end

data/config/initializers/inflections.rb

@@ -12,6 +12,8 @@ module EasyML
           inflect.acronym "EST"
           inflect.acronym "UTC"
           inflect.acronym "HTML"
+          inflect.acronym "API"
+          inflect.acronym "APIs"
         end
       end
     end

data/config/routes.rb

@@ -11,6 +11,9 @@ EasyML::Engine.routes.draw do
   # Predictions API
   resources :predictions, only: [:create]
 
+  # API Documentation
+  get "api", to: "apis#show"
+
   resources :models, as: :easy_ml_models do
     member do
       post :train

data/lib/easy_ml/core/tuner.rb

@@ -173,7 +173,7 @@ module EasyML
         end
         raise ArgumentError, "Objectives required for EasyML::Core::Tuner" unless objective.present?
 
-        self.metrics = EasyML::Model.new(task: task).allowed_metrics if metrics.nil? || metrics.empty?
+        self.metrics = EasyML::Model.new(task: task).default_metrics if metrics.nil? || metrics.empty?
       end
     end
   end

data/lib/easy_ml/data/date_converter.rb

@@ -3,35 +3,104 @@ module EasyML
     module DateConverter
       COMMON_DATE_FORMATS = [
         "%Y-%m-%dT%H:%M:%S.%6N",   # e.g., "2021-01-01T00:00:00.000000"
-        "%Y-%m-%d %H:%M:%S.%L Z", # e.g., "2025-01-03 23:04:49.492 Z"
-        "%Y-%m-%d %H:%M:%S.%L",    # e.g., "2021-01-01 00:01:36.000"
-        "%Y-%m-%d %H:%M:%S.%L",   # e.g., "2021-01-01 00:01:36.000"
-        "%Y-%m-%d %H:%M:%S",      # e.g., "2021-01-01 00:01:36"
-        "%Y-%m-%d %H:%M",         # e.g., "2021-01-01 00:01"
-        "%Y-%m-%d",               # e.g., "2021-01-01"
-        "%m/%d/%Y %H:%M:%S",      # e.g., "01/01/2021 00:01:36"
-        "%m/%d/%Y",               # e.g., "01/01/2021"
-        "%d-%m-%Y",               # e.g., "01-01-2021"
-        "%d-%b-%Y %H:%M:%S",      # e.g., "01-Jan-2021 00:01:36"
-        "%d-%b-%Y",               # e.g., "01-Jan-2021"
-        "%b %d, %Y",              # e.g., "Jan 01, 2021"
-        "%Y/%m/%d %H:%M:%S",      # e.g., "2021/01/01 00:01:36"
+        "%Y-%m-%d %H:%M:%S.%L Z",   # e.g., "2025-01-03 23:04:49.492 Z"
+        "%Y-%m-%d %H:%M:%S.%L",     # e.g., "2021-01-01 00:01:36.000"
+        "%Y-%m-%d %H:%M:%S.%L",     # duplicate format intentionally
+        "%Y-%m-%d %H:%M:%S",        # e.g., "2021-01-01 00:01:36"
+        "%Y-%m-%d %H:%M",           # e.g., "2021-01-01 00:01"
+        "%Y-%m-%d",                 # e.g., "2021-01-01"
+        "%m/%d/%Y %H:%M:%S",        # e.g., "01/01/2021 00:01:36"
+        "%m/%d/%Y",                 # e.g., "01/01/2021"
+        "%d-%m-%Y",                 # e.g., "01-01-2021"
+        "%d-%b-%Y %H:%M:%S",        # e.g., "01-Jan-2021 00:01:36"
+        "%d-%b-%Y",                # e.g., "01-Jan-2021"
+        "%b %d, %Y",               # e.g., "Jan 01, 2021"
+        "%Y/%m/%d %H:%M:%S",        # e.g., "2021/01/01 00:01:36"
         "%Y/%m/%d",                # e.g., "2021/01/01"
       ].freeze
 
       FORMAT_MAPPINGS = {
         ruby_to_polars: {
           "%L" => "%3f",  # milliseconds
-          "%6N" => "%6f", # microseconds
-          "%N" => "%9f", # nanoseconds
+          "%6N" => "%6f",  # microseconds
+          "%N" => "%9f",  # nanoseconds
         },
       }.freeze
 
       class << self
-        # Attempts to convert a string column to datetime if it appears to be a date
-        # @param df [Polars::DataFrame] The dataframe containing the series
-        # @param column [String] The name of the column to convert
-        # @return [Polars::DataFrame] The dataframe with converted column (if successful)
+        # Infers a strftime format string from the given date string.
+        #
+        # @param date_str [String] The date string to analyze.
+        # @return [String, nil] The corresponding strftime format if recognized, or nil if not.
+        def infer_strftime_format(date_str)
+          return nil if date_str.blank?
+
+          # YYYY-MM-DD (e.g., "2021-01-01")
+          return "%Y-%m-%d" if date_str =~ /^\d{4}-\d{2}-\d{2}$/
+
+          # YYYY/MM/DD (e.g., "2021/01/01")
+          return "%Y/%m/%d" if date_str =~ /^\d{4}\/\d{2}\/\d{2}$/
+
+          # Date & time with T separator (ISO 8601-like)
+          if date_str.include?("T")
+            # Without fractional seconds, e.g., "2021-01-01T12:34:56"
+            return "%Y-%m-%dT%H:%M:%S" if date_str =~ /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}$/
+
+            # With fractional seconds, e.g., "2021-01-01T12:34:56.789" or "2021-01-01T12:34:56.123456"
+            if date_str =~ /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.(\d+)$/
+              fraction = Regexp.last_match(1)
+              case fraction.length
+              when 3 then return "%Y-%m-%dT%H:%M:%S.%L"  # milliseconds
+              when 6 then return "%Y-%m-%dT%H:%M:%S.%6N" # microseconds
+              when 9 then return "%Y-%m-%dT%H:%M:%S.%N"  # nanoseconds
+              else
+                # Fallback if fractional part has unexpected length:
+                return "%Y-%m-%dT%H:%M:%S.%N"
+              end
+            end
+          end
+
+          # Date & time with space separator
+          if date_str.include?(" ")
+            # Without fractional seconds, e.g., "2021-01-01 12:34:56"
+            return "%Y-%m-%d %H:%M:%S" if date_str =~ /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/
+
+            # With fractional seconds, e.g., "2021-01-01 12:34:56.789"
+            if date_str =~ /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.(\d+)$/
+              fraction = Regexp.last_match(1)
+              case fraction.length
+              when 3 then return "%Y-%m-%d %H:%M:%S.%L"
+              when 6 then return "%Y-%m-%d %H:%M:%S.%6N"
+              when 9 then return "%Y-%m-%d %H:%M:%S.%N"
+              else
+                return "%Y-%m-%d %H:%M:%S.%N"
+              end
+            end
+          end
+
+          # Common US-style formats
+
+          # MM/DD/YYYY (e.g., "01/31/2021")
+          return "%m/%d/%Y" if date_str =~ /^\d{2}\/\d{2}\/\d{4}$/
+
+          # DD-MM-YYYY (e.g., "31-01-2021")
+          return "%d-%m-%Y" if date_str =~ /^\d{2}-\d{2}-\d{4}$/
+
+          # DD-Mon-YYYY (e.g., "31-Jan-2021")
+          return "%d-%b-%Y" if date_str =~ /^\d{2}-[A-Za-z]{3}-\d{4}$/
+
+          # Mon DD, YYYY (e.g., "Jan 31, 2021")
+          return "%b %d, %Y" if date_str =~ /^[A-Za-z]{3} \d{2}, \d{4}$/
+
+          # Could add additional heuristics as needed...
+
+          nil  # Return nil if no known format matches.
+        end
+
+        # Attempts to convert a string column to datetime if it appears to be a date.
+        # @param df [Polars::DataFrame] The dataframe containing the series.
+        # @param column [String] The name of the column to convert.
+        # @return [Polars::DataFrame] The dataframe with the converted column (if successful).
         def maybe_convert_date(df, column = nil)
           if column.nil?
             series = df
@@ -40,19 +109,42 @@ module EasyML
           else
             series = df[column]
           end
+
           return df if series.dtype.is_a?(Polars::Datetime)
           return df unless series.dtype == Polars::Utf8
 
-          format = detect_polars_format(series)
-          return df unless format
+          sample = series.filter(series.is_not_null).head(100).to_a
+          ruby_format = detect_date_format(sample)
 
-          df.with_column(
-            Polars.col(column.to_s).str.strptime(Polars::Datetime, format).alias(column.to_s)
-          )
+          if ruby_format
+            format = convert_format(:ruby_to_polars, ruby_format)
+            df = try_format(df, column, format)
+
+            if df.filter(Polars.col("TRY").is_null).count > df.filter(Polars.col(column.to_s).is_null).count
+              df = df.drop("TRY")
+              best_format = df[column.to_s][0..100].to_a.count_by do |date_str|
+                infer_strftime_format(date_str)
+              end.max_by { |_format, count| count }[0]
+              df = try_format(df, column, best_format)
+            end
+
+            df = df.with_column(df["TRY"].alias(column.to_s)).drop("TRY")
+          end
+
+          df
         end
 
         private
 
+        def try_format(df, column, format)
+          df = df.with_column(
+            Polars.col(column.to_s)
+              .str
+              .strptime(Polars::Datetime, format, strict: false)
+              .alias("TRY")
+          )
+        end
+
         def detect_polars_format(series)
           return nil unless series.is_a?(Polars::Series)
 
@@ -66,14 +158,29 @@ module EasyML
 
           sample = date_strings.compact.sample([100, date_strings.length].min)
 
-          COMMON_DATE_FORMATS.detect do |format|
-            sample.all? do |date_str|
-              DateTime.strptime(date_str, format)
-              true
-            rescue StandardError
-              false
+          best_format = nil
+          best_success_rate = 0.0
+          sample_count = sample.length
+
+          COMMON_DATE_FORMATS.each do |fmt|
+            success_count = sample.count do |date_str|
+              begin
+                DateTime.strptime(date_str, fmt)
+                true
+              rescue StandardError
+                false
+              end
             end
+            success_rate = success_count.to_f / sample_count
+            if success_rate > best_success_rate
+              best_success_rate = success_rate
+              best_format = fmt
+            end
+            # If every sample string matches this format, return it immediately.
+            return fmt if success_rate == 1.0
           end
+
+          best_success_rate >= 0.8 ? best_format : nil
         end
 
         def convert_format(conversion, format)

data/lib/easy_ml/data/polars_column.rb

@@ -12,6 +12,7 @@ module EasyML
         string: Polars::String,
         text: Polars::String,
         categorical: Polars::Categorical,
+        null: Polars::Null,
       }
       POLARS_MAP = TYPE_MAP.invert.stringify_keys
       class << self
@@ -19,6 +20,20 @@ module EasyML
           POLARS_MAP.dig(polars_type.class.to_s)
         end
 
+        def parse_polars_dtype(dtype_string)
+          case dtype_string
+          when /^Polars::Datetime/
+            time_unit = dtype_string[/time_unit: "(.*?)"/, 1]
+            time_zone = dtype_string[/time_zone: (.*)?\)/, 1]
+            time_zone = time_zone == "nil" ? nil : time_zone&.delete('"')
+            Polars::Datetime.new(time_unit, time_zone)
+          when /^Polars::/
+            Polars.const_get(dtype_string.split("::").last)
+          else
+            raise ArgumentError, "Unknown Polars data type: #{dtype_string}"
+          end
+        end
+
         def sym_to_polars(symbol)
           TYPE_MAP.dig(symbol)
         end
@@ -50,6 +65,8 @@ module EasyML
               :boolean
             when Polars::Utf8
               determine_string_type(series)
+            when Polars::Null
+              :null
             else
               :categorical
             end

data/lib/easy_ml/data/polars_in_memory.rb

@@ -0,0 +1,30 @@
+module EasyML
+  module Data
+    class PolarsInMemory
+      attr_reader :df
+
+      def initialize(df)
+        @df = df
+      end
+
+      def self.query(df, **kwargs)
+        new(df).query(**kwargs)
+      end
+
+      def query(drop_cols: [], filter: nil, limit: nil, select: nil, unique: nil, sort: nil, descending: false)
+        return if df.nil?
+
+        df = self.df.clone
+        df = df.filter(filter) if filter
+        select = df.columns & ([select] || []).flatten
+        df = df.select(select) if select.present?
+        df = df.unique if unique
+        drop_cols &= df.columns
+        df = df.drop(drop_cols) unless drop_cols.empty?
+        df = df.sort(sort, reverse: descending) if sort
+        df = df.limit(limit) if limit
+        df
+      end
+    end
+  end
+end

data/lib/easy_ml/data/polars_reader.rb

@@ -12,6 +12,22 @@ module EasyML
         @schema = options[:schema]
       end
 
+      def sha
+        files = parquet_files.sort
+
+        file_hashes = files.map do |file|
+          meta = Polars.read_parquet_schema(file)
+          row_count = Polars.scan_parquet(file).select(Polars.col("*").count).collect[0, 0]
+
+          Digest::SHA256.hexdigest([
+            meta.to_json,
+            row_count.to_s,
+          ].join("|"))
+        end
+
+        Digest::SHA256.hexdigest(file_hashes.join)
+      end
+
       def schema=(value)
         @schema = value
         polars_args[:dtypes] = value
@@ -55,7 +71,10 @@ module EasyML
         return files if any_parquet? && columns.nil?
 
         puts "Converting to Parquet..."
-
+        if columns.nil? || columns.all? { |c| c.datatype.nil? }
+          learn_dataset
+          columns = nil
+        end
         csv_files.each do |path|
           df = read_file(path, columns)
           df = cast(df, columns)