easy_ml 0.2.0.pre.rc39 → 0.2.0.pre.rc41

data/lib/easy_ml/data/polars_reader.rb

@@ -196,15 +196,22 @@ module EasyML
           polars_args[:dtypes].merge!(dtypes)
         end
         ext = Pathname.new(file).extname.gsub(/\./, "")
+        date_cols = []
         case ext
         when "csv"
-          filtered_args = filter_polars_args(Polars.method(:read_csv))
-          filtered_args.merge!(infer_schema_length: 1_000_000, null_values: ["\\N", "\\\\N", "NULL"])
+          filtered_args, date_cols = filter_polars_args(Polars.method(:read_csv))
+          filtered_args.merge!(
+            infer_schema_length: 1_000_000,
+            null_values: ["\\N", "\\\\N", "NULL"],
+          )
           df = Polars.read_csv(file, **filtered_args)
         when "parquet"
-          filtered_args = filter_polars_args(Polars.method(:read_parquet))
+          filtered_args, date_cols = filter_polars_args(Polars.method(:read_parquet))
           df = Polars.read_parquet(file, **filtered_args)
         end
+        date_cols.each do |col|
+          df = EasyML::Data::DateConverter.maybe_convert_date(df, col)
+        end
         df
       end
 
@@ -214,7 +221,13 @@ module EasyML
 
       def filter_polars_args(method)
         supported_params = method.parameters.map { |_, name| name }
-        polars_args.select { |k, _| supported_params.include?(k) }
+        filtered = polars_args.select { |k, _| supported_params.include?(k) }
+
+        # Filter out any datetime columns, and use maybe_convert_date to convert later
+        date_cols = (filtered[:dtypes] || {}).select { |k, v| v.class == Polars::Datetime }.keys
+        filtered[:dtypes] = (filtered[:dtypes] || {}).reject { |k, v| v.class == Polars::Datetime }
+        filtered = filtered.select { |k, _| supported_params.include?(k) }
+        return filtered, date_cols
       end
 
       def csv_files

data/lib/easy_ml/data/statistics_learner.rb

@@ -59,7 +59,7 @@ module EasyML::Data
           stats[col].merge!(most_frequent_value: series.mode.sort.to_a&.first)
           if field_type == :categorical
             stats[col].merge!(
-              unique_count: series.n_unique,
+              unique_count: series.cast(:str).n_unique,
               counts: Hash[series.value_counts.to_hashes.map(&:values)],
             )
           end

data/lib/easy_ml/engine.rb

@@ -1,5 +1,6 @@
 require "aws-sdk"
 require "awesome_print"
+require "rails/all"
 require "inertia_rails"
 require "jsonapi/serializer"
 require "numo/narray"
@@ -68,6 +69,16 @@ module EasyML
       end
     end
 
+    initializer "easy_ml.check_pending_migrations" do
+      if defined?(Rails::Server)
+        config.after_initialize do
+          if EasyML.pending_migrations?
+            puts "\e[33mWARNING: You have pending EasyML migrations. Run 'rails generate easy_ml:migration' to add them.\e[0m"
+          end
+        end
+      end
+    end
+
     initializer "easy_ml.active_job_config" do
       resque_initializer = File.expand_path("config/initializers/resque.rb", root)
       require resque_initializer if File.exist?(resque_initializer)
@@ -87,7 +98,7 @@ module EasyML
       end
     end
 
-    if ENV["EASY_ML_DEMO_APP"]
+    if ENV["EASY_ML_DEV"]
       require "vite_ruby"
       require "vite_rails"
 

data/lib/easy_ml/pending_migrations.rb

@@ -0,0 +1,19 @@
+module EasyML
+  def self.pending_migrations?
+    return false unless defined?(ActiveRecord)
+
+    # Get all migration files from our templates
+    template_dir = File.expand_path("../railtie/generators/templates/migration", __dir__)
+    template_migrations = Dir.glob(File.join(template_dir, "*.tt")).map do |f|
+      File.basename(f, ".tt").sub(/^create_/, "")
+    end
+
+    # Get all existing migrations
+    existing_migrations = Dir.glob(Rails.root.join("db/migrate/*_*.rb")).map do |f|
+      File.basename(f).sub(/^\d+_create_/, "").sub(/\.rb$/, "")
+    end
+
+    # Check if any template migrations are not in existing migrations
+    (template_migrations - existing_migrations).any?
+  end
+end

data/lib/easy_ml/predict.rb

@@ -23,9 +23,7 @@ module EasyML
         model: current_version.model,
         model_history: current_version,
         prediction_type: current_version.model.task,
-        prediction_value: {
-          value: preds.first,
-        }.compact,
+        prediction_value: preds.first,
         raw_input: raw_input,
         normalized_input: df.to_hashes&.first,
       )

data/lib/easy_ml/railtie/generators/migration/migration_generator.rb

@@ -15,6 +15,32 @@ module EasyML
           # Define the migration name
           desc "Generates migrations for EasyMLModel"
 
+          # Define the order of migrations
+          MIGRATION_ORDER = %w[
+            create_easy_ml_datasources
+            create_easy_ml_datasets
+            create_easy_ml_columns
+            create_easy_ml_models
+            create_easy_ml_model_files
+            create_easy_ml_tuner_jobs
+            create_easy_ml_retraining_jobs
+            create_easy_ml_settings
+            create_easy_ml_events
+            create_easy_ml_features
+            create_easy_ml_splitters
+            create_easy_ml_splitter_histories
+            create_easy_ml_deploys
+            create_easy_ml_datasource_histories
+            create_easy_ml_dataset_histories
+            create_easy_ml_column_histories
+            create_easy_ml_model_histories
+            create_easy_ml_model_file_histories
+            create_easy_ml_feature_histories
+            create_easy_ml_predictions
+            create_easy_ml_event_contexts
+            add_workflow_status_to_easy_ml_features
+          ].freeze
+
           # Specify the next migration number
           def self.next_migration_number(dirname)
             sleep(1)
@@ -29,169 +55,24 @@ module EasyML
 
           # Generate the migration files using the templates
           def create_migration_files
-            create_easy_ml_datasource_migration
-            create_easy_ml_datasets_migration
-            create_easy_ml_columns_migration
-            create_easy_ml_models_migration
-            create_easy_ml_model_files_migration
-            create_easy_ml_tuner_jobs_migration
-            create_easy_ml_retraining_jobs_migration
-            create_easy_ml_settings_migration
-            create_easy_ml_events_migration
-            create_easy_ml_features_migration
-            create_easy_ml_splitters_migration
-            create_easy_ml_splitter_histories_migration
-            create_easy_ml_deploys
+            # Check for existing migrations first
+            existing_migrations = Dir.glob(Rails.root.join("db/migrate/*_*.rb")).map do |f|
+              File.basename(f).sub(/^\d+_/, "").sub(/\.rb$/, "")
+            end
 
-            create_easy_ml_datasource_histories_migration
-            create_easy_ml_dataset_histories_migration
-            create_easy_ml_column_histories_migration
-            create_easy_ml_model_histories_migration
-            create_easy_ml_model_file_histories_migration
-            create_easy_ml_feature_histories_migration
-            create_easy_ml_predictions_migration
+            # Create migrations in order if they don't exist
+            MIGRATION_ORDER.each do |migration_name|
+              next if existing_migrations.include?(migration_name)
+              install_migration(migration_name)
+            end
           end
 
           private
 
-          # Generate the migration file for EasyMLModel using the template
-          def create_easy_ml_models_migration
-            migration_template(
-              "create_easy_ml_models.rb.tt",
-              "db/migrate/create_easy_ml_models.rb"
-            )
-          end
-
-          def create_easy_ml_model_files_migration
-            migration_template(
-              "create_easy_ml_model_files.rb.tt",
-              "db/migrate/create_easy_ml_model_files.rb"
-            )
-          end
-
-          def create_easy_ml_datasource_migration
-            migration_template(
-              "create_easy_ml_datasources.rb.tt",
-              "db/migrate/create_easy_ml_datasources.rb"
-            )
-          end
-
-          def create_easy_ml_datasets_migration
-            migration_template(
-              "create_easy_ml_datasets.rb.tt",
-              "db/migrate/create_easy_ml_datasets.rb"
-            )
-          end
-
-          def create_easy_ml_tuner_jobs_migration
-            migration_template(
-              "create_easy_ml_tuner_jobs.rb.tt",
-              "db/migrate/create_easy_ml_tuner_jobs.rb"
-            )
-          end
-
-          def create_easy_ml_retraining_jobs_migration
-            migration_template(
-              "create_easy_ml_retraining_jobs.rb.tt",
-              "db/migrate/create_easy_ml_retraining_jobs.rb"
-            )
-          end
-
-          def create_easy_ml_settings_migration
-            migration_template(
-              "create_easy_ml_settings.rb.tt",
-              "db/migrate/create_easy_ml_settings.rb"
-            )
-          end
-
-          def create_easy_ml_events_migration
-            migration_template(
-              "create_easy_ml_events.rb.tt",
-              "db/migrate/create_easy_ml_events.rb"
-            )
-          end
-
-          def create_easy_ml_columns_migration
-            migration_template(
-              "create_easy_ml_columns.rb.tt",
-              "db/migrate/create_easy_ml_columns.rb"
-            )
-          end
-
-          def create_easy_ml_features_migration
-            migration_template(
-              "create_easy_ml_features.rb.tt",
-              "db/migrate/create_easy_ml_features.rb"
-            )
-          end
-
-          def create_easy_ml_splitters_migration
-            migration_template(
-              "create_easy_ml_splitters.rb.tt",
-              "db/migrate/create_easy_ml_splitters.rb"
-            )
-          end
-
-          def create_easy_ml_splitter_histories_migration
-            migration_template(
-              "create_easy_ml_splitter_histories.rb.tt",
-              "db/migrate/create_easy_ml_splitter_histories.rb"
-            )
-          end
-
-          def create_easy_ml_datasource_histories_migration
-            migration_template(
-              "create_easy_ml_datasource_histories.rb.tt",
-              "db/migrate/create_easy_ml_datasource_histories.rb"
-            )
-          end
-
-          def create_easy_ml_dataset_histories_migration
-            migration_template(
-              "create_easy_ml_dataset_histories.rb.tt",
-              "db/migrate/create_easy_ml_dataset_histories.rb"
-            )
-          end
-
-          def create_easy_ml_column_histories_migration
-            migration_template(
-              "create_easy_ml_column_histories.rb.tt",
-              "db/migrate/create_easy_ml_column_histories.rb"
-            )
-          end
-
-          def create_easy_ml_model_histories_migration
-            migration_template(
-              "create_easy_ml_model_histories.rb.tt",
-              "db/migrate/create_easy_ml_model_histories.rb"
-            )
-          end
-
-          def create_easy_ml_feature_histories_migration
-            migration_template(
-              "create_easy_ml_feature_histories.rb.tt",
-              "db/migrate/create_easy_ml_feature_histories.rb"
-            )
-          end
-
-          def create_easy_ml_model_file_histories_migration
-            migration_template(
-              "create_easy_ml_model_file_histories.rb.tt",
-              "db/migrate/create_easy_ml_model_file_histories.rb"
-            )
-          end
-
-          def create_easy_ml_deploys
-            migration_template(
-              "create_easy_ml_deploys.rb.tt",
-              "db/migrate/create_easy_ml_deploys.rb"
-            )
-          end
-
-          def create_easy_ml_predictions_migration
+          def install_migration(migration_name)
             migration_template(
-              "create_easy_ml_predictions.rb.tt",
-              "db/migrate/create_easy_ml_predictions.rb"
+              "#{migration_name}.rb.tt",
+              "db/migrate/#{migration_name}.rb"
             )
           end
 

data/lib/easy_ml/railtie/templates/migration/add_workflow_status_to_easy_ml_features.rb.tt

@@ -0,0 +1,8 @@
+class AddWorkflowStatusToEasyMLFeatures < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    unless column_exists?(:easy_ml_features, :workflow_status)
+      add_column :easy_ml_features, :workflow_status, :string
+      add_index :easy_ml_features, :workflow_status
+    end
+  end
+end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_column_histories.rb.tt

@@ -2,8 +2,10 @@ require "historiographer/postgres_migration"
 
 class CreateEasyMLColumnHistories < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
   def change
-    create_table :easy_ml_column_histories do |t|
-      t.histories(foreign_key: :column_id)
+    unless table_exists?(:easy_ml_column_histories)
+      create_table :easy_ml_column_histories do |t|
+        t.histories(foreign_key: :column_id)
+      end
     end
   end
 end 

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_columns.rb.tt

@@ -1,25 +1,27 @@
 class CreateEasyMLColumns < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
   def change
-    create_table :easy_ml_columns do |t|
-      t.bigint :dataset_id, null: false
-      t.string :name, null: false
-      t.string :description
-      t.string :datatype # The symbol representation (e.g., 'float', 'integer')
-      t.string :polars_datatype # The full Polars class name (e.g., 'Polars::Float64')
-      t.boolean :is_target
-      t.boolean :hidden, default: false
-      t.boolean :drop_if_null, default: false
-      t.json :preprocessing_steps
-      t.json :sample_values # Store up to 3 sample values
-      t.json :statistics
+    unless table_exists?(:easy_ml_columns)
+      create_table :easy_ml_columns do |t|
+        t.bigint :dataset_id, null: false
+        t.string :name, null: false
+        t.string :description
+        t.string :datatype # The symbol representation (e.g., 'float', 'integer')
+        t.string :polars_datatype # The full Polars class name (e.g., 'Polars::Float64')
+        t.boolean :is_target
+        t.boolean :hidden, default: false
+        t.boolean :drop_if_null, default: false
+        t.json :preprocessing_steps
+        t.json :sample_values # Store up to 3 sample values
+        t.json :statistics
 
-      t.timestamps
+        t.timestamps
 
-      t.index [:dataset_id, :name], unique: true
-      t.index :datatype
-      t.index :hidden
-      t.index :drop_if_null
-      t.index :is_target
+        t.index [:dataset_id, :name], unique: true
+        t.index :datatype
+        t.index :hidden
+        t.index :drop_if_null
+        t.index :is_target
+      end
     end
-  end
-end 
+  end 
+end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_dataset_histories.rb.tt

@@ -2,8 +2,10 @@ require "historiographer/postgres_migration"
 
 class CreateEasyMLDatasetHistories < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
   def change
-    create_table :easy_ml_dataset_histories do |t|
-      t.histories(foreign_key: :dataset_id)
+    unless table_exists?(:easy_ml_dataset_histories)
+      create_table :easy_ml_dataset_histories do |t|
+        t.histories(foreign_key: :dataset_id)
+      end
     end
   end
-end 
+end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_datasets.rb.tt

@@ -1,31 +1,33 @@
 class CreateEasyMLDatasets < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
   def change
-    create_table :easy_ml_datasets do |t|
-      t.string :name, null: false
-      t.string :description
-      t.string :dataset_type
-      t.string :status
-      t.string :version
-      t.bigint :datasource_id
-      t.string :root_dir
-      t.json :configuration
-      t.bigint :num_rows
-      t.string :workflow_status
-      t.json :statistics
-      t.json :preprocessor_statistics
-      t.json :schema
-      t.datetime :refreshed_at
+    unless table_exists?(:easy_ml_datasets)
+      create_table :easy_ml_datasets do |t|
+        t.string :name, null: false
+        t.string :description
+        t.string :dataset_type
+        t.string :status
+        t.string :version
+        t.bigint :datasource_id
+        t.string :root_dir
+        t.json :configuration
+        t.bigint :num_rows
+        t.string :workflow_status
+        t.json :statistics
+        t.json :preprocessor_statistics
+        t.json :schema
+        t.datetime :refreshed_at
 
-      t.timestamps
+        t.timestamps
 
-      t.index :created_at
-      t.index :refreshed_at
-      t.index :name
-      t.index :status
-      t.index [:name, :status]
-      t.index :datasource_id
-      t.index :dataset_type
-      t.index :workflow_status
+        t.index :created_at
+        t.index :refreshed_at
+        t.index :name
+        t.index :status
+        t.index [:name, :status]
+        t.index :datasource_id
+        t.index :dataset_type
+        t.index :workflow_status
+      end
     end
   end
 end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_datasource_histories.rb.tt

@@ -2,8 +2,10 @@ require "historiographer/postgres_migration"
 
 class CreateEasyMLDatasourceHistories < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
   def change
-    create_table :easy_ml_datasource_histories do |t|
-      t.histories(foreign_key: :datasource_id)
+    unless table_exists?(:easy_ml_datasource_histories)
+      create_table :easy_ml_datasource_histories do |t|
+        t.histories(foreign_key: :datasource_id)
+      end
     end
   end
-end 
+end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_datasources.rb.tt

@@ -1,16 +1,18 @@
 class CreateEasyMLDatasources < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
   def change
-    create_table :easy_ml_datasources do |t|
-      t.string :name, null: false
-      t.string :datasource_type
-      t.string :root_dir
-      t.json :configuration
-      t.datetime :refreshed_at
+    unless table_exists?(:easy_ml_datasources)
+      create_table :easy_ml_datasources do |t|
+        t.string :name, null: false
+        t.string :datasource_type
+        t.string :root_dir
+        t.json :configuration
+        t.datetime :refreshed_at
 
-      t.timestamps
-      t.index :created_at
-      t.index :datasource_type
-      t.index :refreshed_at
+        t.timestamps
+        t.index :created_at
+        t.index :datasource_type
+        t.index :refreshed_at
+      end
     end
   end
 end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_deploys.rb.tt

@@ -1,24 +1,26 @@
 class CreateEasyMLDeploys < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
   def change
-    create_table :easy_ml_deploys do |t|
-      t.bigint :model_id
-      t.bigint :model_history_id
-      t.bigint :retraining_run_id
-      t.bigint :model_file_id
-      t.string :status, null: false
-      t.string :trigger, default: 'manual'
-      t.text :stacktrace
-      t.string :snapshot_id
-      t.timestamps
+    unless table_exists?(:easy_ml_deploys)
+      create_table :easy_ml_deploys do |t|
+        t.bigint :model_id
+        t.bigint :model_history_id
+        t.bigint :retraining_run_id
+        t.bigint :model_file_id
+        t.string :status, null: false
+        t.string :trigger, default: 'manual'
+        t.text :stacktrace
+        t.string :snapshot_id
+        t.timestamps
 
-      t.index :created_at
-      t.index :model_id
-      t.index :model_history_id
-      t.index :snapshot_id
-      t.index :model_file_id
-      t.index :retraining_run_id
-      t.index :status
-      t.index :trigger
+        t.index :created_at
+        t.index :model_id
+        t.index :model_history_id
+        t.index :snapshot_id
+        t.index :model_file_id
+        t.index :retraining_run_id
+        t.index :status
+        t.index :trigger
+      end
     end
   end
-end 
+end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_event_contexts.rb.tt

@@ -0,0 +1,14 @@
+class CreateEasyMLEventContexts < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    unless table_exists?(:easy_ml_event_contexts)
+      create_table :easy_ml_event_contexts do |t|
+        t.references :event, null: false, foreign_key: { to_table: :easy_ml_events }
+        t.jsonb :context, null: false, default: {}
+        t.string :format
+        t.timestamps
+      end
+
+      add_index :easy_ml_event_contexts, :context, using: :gin
+    end
+  end
+end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_events.rb.tt

@@ -1,20 +1,22 @@
 class CreateEasyMLEvents < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
   def change
-    create_table :easy_ml_events do |t|
-      t.string :name, null: false
-      t.string :status, null: false
-      t.string :eventable_type
-      t.bigint :eventable_id
-      t.text :stacktrace
+    unless table_exists?(:easy_ml_events)
+      create_table :easy_ml_events do |t|
+        t.string :name, null: false
+        t.string :status, null: false
+        t.string :eventable_type
+        t.bigint :eventable_id
+        t.text :stacktrace
 
-      t.timestamps
+        t.timestamps
 
-      t.index :name
-      t.index :status
-      t.index :eventable_type
-      t.index :eventable_id
-      t.index :created_at
-      t.index [:eventable_type, :eventable_id]
+        t.index :name
+        t.index :status
+        t.index :eventable_type
+        t.index :eventable_id
+        t.index :created_at
+        t.index [:eventable_type, :eventable_id]
+      end
     end
   end
-end 
+end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_feature_histories.rb.tt

@@ -2,13 +2,15 @@ require "historiographer/postgres_migration"
 
 class CreateEasyMLFeatureHistories < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
   def change
-    create_table :easy_ml_feature_histories do |t|
-      t.histories(
-        foreign_key: :feature_id,
-        index_names: {
-          [:dataset_id, :feature_position] => "idx_feature_histories_on_dataset_and_position"
-        }
-      )
+    unless table_exists?(:easy_ml_feature_histories)
+      create_table :easy_ml_feature_histories do |t|
+        t.histories(
+          foreign_key: :feature_id,
+          index_names: {
+            [:dataset_id, :feature_position] => "idx_feature_histories_on_dataset_and_position"
+          }
+        )
+      end
     end
   end
-end 
+end