easy_ml 0.2.0.pre.rc40 → 0.2.0.pre.rc43

data/lib/easy_ml/railtie/generators/migration/migration_generator.rb

@@ -15,6 +15,33 @@ module EasyML
           # Define the migration name
           desc "Generates migrations for EasyMLModel"
 
+          # Define the order of migrations
+          MIGRATION_ORDER = %w[
+            create_easy_ml_datasources
+            create_easy_ml_datasets
+            create_easy_ml_columns
+            create_easy_ml_models
+            create_easy_ml_model_files
+            create_easy_ml_tuner_jobs
+            create_easy_ml_retraining_jobs
+            create_easy_ml_settings
+            create_easy_ml_events
+            create_easy_ml_features
+            create_easy_ml_splitters
+            create_easy_ml_splitter_histories
+            create_easy_ml_deploys
+            create_easy_ml_datasource_histories
+            create_easy_ml_dataset_histories
+            create_easy_ml_column_histories
+            create_easy_ml_model_histories
+            create_easy_ml_model_file_histories
+            create_easy_ml_feature_histories
+            create_easy_ml_predictions
+            create_easy_ml_event_contexts
+            add_workflow_status_to_easy_ml_features
+            drop_path_from_easy_ml_model_files
+          ].freeze
+
           # Specify the next migration number
           def self.next_migration_number(dirname)
             sleep(1)
@@ -29,169 +56,24 @@ module EasyML
 
           # Generate the migration files using the templates
           def create_migration_files
-            create_easy_ml_datasource_migration
-            create_easy_ml_datasets_migration
-            create_easy_ml_columns_migration
-            create_easy_ml_models_migration
-            create_easy_ml_model_files_migration
-            create_easy_ml_tuner_jobs_migration
-            create_easy_ml_retraining_jobs_migration
-            create_easy_ml_settings_migration
-            create_easy_ml_events_migration
-            create_easy_ml_features_migration
-            create_easy_ml_splitters_migration
-            create_easy_ml_splitter_histories_migration
-            create_easy_ml_deploys
+            # Check for existing migrations first
+            existing_migrations = Dir.glob(Rails.root.join("db/migrate/*_*.rb")).map do |f|
+              File.basename(f).sub(/^\d+_/, "").sub(/\.rb$/, "")
+            end
 
-            create_easy_ml_datasource_histories_migration
-            create_easy_ml_dataset_histories_migration
-            create_easy_ml_column_histories_migration
-            create_easy_ml_model_histories_migration
-            create_easy_ml_model_file_histories_migration
-            create_easy_ml_feature_histories_migration
-            create_easy_ml_predictions_migration
+            # Create migrations in order if they don't exist
+            MIGRATION_ORDER.each do |migration_name|
+              next if existing_migrations.include?(migration_name)
+              install_migration(migration_name)
+            end
           end
 
           private
 
-          # Generate the migration file for EasyMLModel using the template
-          def create_easy_ml_models_migration
-            migration_template(
-              "create_easy_ml_models.rb.tt",
-              "db/migrate/create_easy_ml_models.rb"
-            )
-          end
-
-          def create_easy_ml_model_files_migration
-            migration_template(
-              "create_easy_ml_model_files.rb.tt",
-              "db/migrate/create_easy_ml_model_files.rb"
-            )
-          end
-
-          def create_easy_ml_datasource_migration
-            migration_template(
-              "create_easy_ml_datasources.rb.tt",
-              "db/migrate/create_easy_ml_datasources.rb"
-            )
-          end
-
-          def create_easy_ml_datasets_migration
-            migration_template(
-              "create_easy_ml_datasets.rb.tt",
-              "db/migrate/create_easy_ml_datasets.rb"
-            )
-          end
-
-          def create_easy_ml_tuner_jobs_migration
-            migration_template(
-              "create_easy_ml_tuner_jobs.rb.tt",
-              "db/migrate/create_easy_ml_tuner_jobs.rb"
-            )
-          end
-
-          def create_easy_ml_retraining_jobs_migration
-            migration_template(
-              "create_easy_ml_retraining_jobs.rb.tt",
-              "db/migrate/create_easy_ml_retraining_jobs.rb"
-            )
-          end
-
-          def create_easy_ml_settings_migration
-            migration_template(
-              "create_easy_ml_settings.rb.tt",
-              "db/migrate/create_easy_ml_settings.rb"
-            )
-          end
-
-          def create_easy_ml_events_migration
-            migration_template(
-              "create_easy_ml_events.rb.tt",
-              "db/migrate/create_easy_ml_events.rb"
-            )
-          end
-
-          def create_easy_ml_columns_migration
-            migration_template(
-              "create_easy_ml_columns.rb.tt",
-              "db/migrate/create_easy_ml_columns.rb"
-            )
-          end
-
-          def create_easy_ml_features_migration
-            migration_template(
-              "create_easy_ml_features.rb.tt",
-              "db/migrate/create_easy_ml_features.rb"
-            )
-          end
-
-          def create_easy_ml_splitters_migration
-            migration_template(
-              "create_easy_ml_splitters.rb.tt",
-              "db/migrate/create_easy_ml_splitters.rb"
-            )
-          end
-
-          def create_easy_ml_splitter_histories_migration
-            migration_template(
-              "create_easy_ml_splitter_histories.rb.tt",
-              "db/migrate/create_easy_ml_splitter_histories.rb"
-            )
-          end
-
-          def create_easy_ml_datasource_histories_migration
-            migration_template(
-              "create_easy_ml_datasource_histories.rb.tt",
-              "db/migrate/create_easy_ml_datasource_histories.rb"
-            )
-          end
-
-          def create_easy_ml_dataset_histories_migration
-            migration_template(
-              "create_easy_ml_dataset_histories.rb.tt",
-              "db/migrate/create_easy_ml_dataset_histories.rb"
-            )
-          end
-
-          def create_easy_ml_column_histories_migration
-            migration_template(
-              "create_easy_ml_column_histories.rb.tt",
-              "db/migrate/create_easy_ml_column_histories.rb"
-            )
-          end
-
-          def create_easy_ml_model_histories_migration
-            migration_template(
-              "create_easy_ml_model_histories.rb.tt",
-              "db/migrate/create_easy_ml_model_histories.rb"
-            )
-          end
-
-          def create_easy_ml_feature_histories_migration
-            migration_template(
-              "create_easy_ml_feature_histories.rb.tt",
-              "db/migrate/create_easy_ml_feature_histories.rb"
-            )
-          end
-
-          def create_easy_ml_model_file_histories_migration
-            migration_template(
-              "create_easy_ml_model_file_histories.rb.tt",
-              "db/migrate/create_easy_ml_model_file_histories.rb"
-            )
-          end
-
-          def create_easy_ml_deploys
-            migration_template(
-              "create_easy_ml_deploys.rb.tt",
-              "db/migrate/create_easy_ml_deploys.rb"
-            )
-          end
-
-          def create_easy_ml_predictions_migration
+          def install_migration(migration_name)
             migration_template(
-              "create_easy_ml_predictions.rb.tt",
-              "db/migrate/create_easy_ml_predictions.rb"
+              "#{migration_name}.rb.tt",
+              "db/migrate/#{migration_name}.rb"
             )
           end
 

data/lib/easy_ml/railtie/templates/migration/add_workflow_status_to_easy_ml_features.rb.tt

@@ -0,0 +1,13 @@
+class AddWorkflowStatusToEasyMLFeatures < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    unless column_exists?(:easy_ml_features, :workflow_status)
+      add_column :easy_ml_features, :workflow_status, :string
+      add_index :easy_ml_features, :workflow_status
+    end
+
+    unless column_exists?(:easy_ml_feature_histories, :workflow_status)
+      add_column :easy_ml_feature_histories, :workflow_status, :string
+      add_index :easy_ml_feature_histories, :workflow_status
+    end
+  end
+end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_column_histories.rb.tt

@@ -2,8 +2,10 @@ require "historiographer/postgres_migration"
 
 class CreateEasyMLColumnHistories < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
   def change
-    create_table :easy_ml_column_histories do |t|
-      t.histories(foreign_key: :column_id)
+    unless table_exists?(:easy_ml_column_histories)
+      create_table :easy_ml_column_histories do |t|
+        t.histories(foreign_key: :column_id)
+      end
     end
   end
 end 

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_columns.rb.tt

@@ -1,25 +1,27 @@
 class CreateEasyMLColumns < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
   def change
-    create_table :easy_ml_columns do |t|
-      t.bigint :dataset_id, null: false
-      t.string :name, null: false
-      t.string :description
-      t.string :datatype # The symbol representation (e.g., 'float', 'integer')
-      t.string :polars_datatype # The full Polars class name (e.g., 'Polars::Float64')
-      t.boolean :is_target
-      t.boolean :hidden, default: false
-      t.boolean :drop_if_null, default: false
-      t.json :preprocessing_steps
-      t.json :sample_values # Store up to 3 sample values
-      t.json :statistics
+    unless table_exists?(:easy_ml_columns)
+      create_table :easy_ml_columns do |t|
+        t.bigint :dataset_id, null: false
+        t.string :name, null: false
+        t.string :description
+        t.string :datatype # The symbol representation (e.g., 'float', 'integer')
+        t.string :polars_datatype # The full Polars class name (e.g., 'Polars::Float64')
+        t.boolean :is_target
+        t.boolean :hidden, default: false
+        t.boolean :drop_if_null, default: false
+        t.json :preprocessing_steps
+        t.json :sample_values # Store up to 3 sample values
+        t.json :statistics
 
-      t.timestamps
+        t.timestamps
 
-      t.index [:dataset_id, :name], unique: true
-      t.index :datatype
-      t.index :hidden
-      t.index :drop_if_null
-      t.index :is_target
+        t.index [:dataset_id, :name], unique: true
+        t.index :datatype
+        t.index :hidden
+        t.index :drop_if_null
+        t.index :is_target
+      end
     end
-  end
-end 
+  end 
+end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_dataset_histories.rb.tt

@@ -2,8 +2,10 @@ require "historiographer/postgres_migration"
 
 class CreateEasyMLDatasetHistories < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
   def change
-    create_table :easy_ml_dataset_histories do |t|
-      t.histories(foreign_key: :dataset_id)
+    unless table_exists?(:easy_ml_dataset_histories)
+      create_table :easy_ml_dataset_histories do |t|
+        t.histories(foreign_key: :dataset_id)
+      end
     end
   end
-end 
+end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_datasets.rb.tt

@@ -1,31 +1,33 @@
 class CreateEasyMLDatasets < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
   def change
-    create_table :easy_ml_datasets do |t|
-      t.string :name, null: false
-      t.string :description
-      t.string :dataset_type
-      t.string :status
-      t.string :version
-      t.bigint :datasource_id
-      t.string :root_dir
-      t.json :configuration
-      t.bigint :num_rows
-      t.string :workflow_status
-      t.json :statistics
-      t.json :preprocessor_statistics
-      t.json :schema
-      t.datetime :refreshed_at
+    unless table_exists?(:easy_ml_datasets)
+      create_table :easy_ml_datasets do |t|
+        t.string :name, null: false
+        t.string :description
+        t.string :dataset_type
+        t.string :status
+        t.string :version
+        t.bigint :datasource_id
+        t.string :root_dir
+        t.json :configuration
+        t.bigint :num_rows
+        t.string :workflow_status
+        t.json :statistics
+        t.json :preprocessor_statistics
+        t.json :schema
+        t.datetime :refreshed_at
 
-      t.timestamps
+        t.timestamps
 
-      t.index :created_at
-      t.index :refreshed_at
-      t.index :name
-      t.index :status
-      t.index [:name, :status]
-      t.index :datasource_id
-      t.index :dataset_type
-      t.index :workflow_status
+        t.index :created_at
+        t.index :refreshed_at
+        t.index :name
+        t.index :status
+        t.index [:name, :status]
+        t.index :datasource_id
+        t.index :dataset_type
+        t.index :workflow_status
+      end
     end
   end
 end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_datasource_histories.rb.tt

@@ -2,8 +2,10 @@ require "historiographer/postgres_migration"
 
 class CreateEasyMLDatasourceHistories < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
   def change
-    create_table :easy_ml_datasource_histories do |t|
-      t.histories(foreign_key: :datasource_id)
+    unless table_exists?(:easy_ml_datasource_histories)
+      create_table :easy_ml_datasource_histories do |t|
+        t.histories(foreign_key: :datasource_id)
+      end
     end
   end
-end 
+end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_datasources.rb.tt

@@ -1,16 +1,18 @@
 class CreateEasyMLDatasources < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
   def change
-    create_table :easy_ml_datasources do |t|
-      t.string :name, null: false
-      t.string :datasource_type
-      t.string :root_dir
-      t.json :configuration
-      t.datetime :refreshed_at
+    unless table_exists?(:easy_ml_datasources)
+      create_table :easy_ml_datasources do |t|
+        t.string :name, null: false
+        t.string :datasource_type
+        t.string :root_dir
+        t.json :configuration
+        t.datetime :refreshed_at
 
-      t.timestamps
-      t.index :created_at
-      t.index :datasource_type
-      t.index :refreshed_at
+        t.timestamps
+        t.index :created_at
+        t.index :datasource_type
+        t.index :refreshed_at
+      end
     end
   end
 end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_deploys.rb.tt

@@ -1,24 +1,26 @@
 class CreateEasyMLDeploys < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
   def change
-    create_table :easy_ml_deploys do |t|
-      t.bigint :model_id
-      t.bigint :model_history_id
-      t.bigint :retraining_run_id
-      t.bigint :model_file_id
-      t.string :status, null: false
-      t.string :trigger, default: 'manual'
-      t.text :stacktrace
-      t.string :snapshot_id
-      t.timestamps
+    unless table_exists?(:easy_ml_deploys)
+      create_table :easy_ml_deploys do |t|
+        t.bigint :model_id
+        t.bigint :model_history_id
+        t.bigint :retraining_run_id
+        t.bigint :model_file_id
+        t.string :status, null: false
+        t.string :trigger, default: 'manual'
+        t.text :stacktrace
+        t.string :snapshot_id
+        t.timestamps
 
-      t.index :created_at
-      t.index :model_id
-      t.index :model_history_id
-      t.index :snapshot_id
-      t.index :model_file_id
-      t.index :retraining_run_id
-      t.index :status
-      t.index :trigger
+        t.index :created_at
+        t.index :model_id
+        t.index :model_history_id
+        t.index :snapshot_id
+        t.index :model_file_id
+        t.index :retraining_run_id
+        t.index :status
+        t.index :trigger
+      end
     end
   end
-end 
+end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_event_contexts.rb.tt

@@ -0,0 +1,14 @@
+class CreateEasyMLEventContexts < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    unless table_exists?(:easy_ml_event_contexts)
+      create_table :easy_ml_event_contexts do |t|
+        t.references :event, null: false, foreign_key: { to_table: :easy_ml_events }
+        t.jsonb :context, null: false, default: {}
+        t.string :format
+        t.timestamps
+      end
+
+      add_index :easy_ml_event_contexts, :context, using: :gin
+    end
+  end
+end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_events.rb.tt

@@ -1,20 +1,22 @@
 class CreateEasyMLEvents < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
   def change
-    create_table :easy_ml_events do |t|
-      t.string :name, null: false
-      t.string :status, null: false
-      t.string :eventable_type
-      t.bigint :eventable_id
-      t.text :stacktrace
+    unless table_exists?(:easy_ml_events)
+      create_table :easy_ml_events do |t|
+        t.string :name, null: false
+        t.string :status, null: false
+        t.string :eventable_type
+        t.bigint :eventable_id
+        t.text :stacktrace
 
-      t.timestamps
+        t.timestamps
 
-      t.index :name
-      t.index :status
-      t.index :eventable_type
-      t.index :eventable_id
-      t.index :created_at
-      t.index [:eventable_type, :eventable_id]
+        t.index :name
+        t.index :status
+        t.index :eventable_type
+        t.index :eventable_id
+        t.index :created_at
+        t.index [:eventable_type, :eventable_id]
+      end
     end
   end
-end 
+end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_feature_histories.rb.tt

@@ -2,13 +2,15 @@ require "historiographer/postgres_migration"
 
 class CreateEasyMLFeatureHistories < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
   def change
-    create_table :easy_ml_feature_histories do |t|
-      t.histories(
-        foreign_key: :feature_id,
-        index_names: {
-          [:dataset_id, :feature_position] => "idx_feature_histories_on_dataset_and_position"
-        }
-      )
+    unless table_exists?(:easy_ml_feature_histories)
+      create_table :easy_ml_feature_histories do |t|
+        t.histories(
+          foreign_key: :feature_id,
+          index_names: {
+            [:dataset_id, :feature_position] => "idx_feature_histories_on_dataset_and_position"
+          }
+        )
+      end
     end
   end
-end 
+end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_features.rb.tt

@@ -1,32 +1,34 @@
 class CreateEasyMLFeatures < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
   def change
-    create_table :easy_ml_features do |t|
-      t.bigint :dataset_id, null: false
-      t.string :name
-      t.bigint :version
-      t.string :feature_class, null: false
-      t.integer :feature_position
-      t.integer :batch_size
-      t.boolean :needs_fit
-      t.string :sha
-      t.string :primary_key, array: true
-      t.datetime :applied_at
-      t.datetime :fit_at
-      t.bigint :refresh_every
+    unless table_exists?(:easy_ml_features)
+      create_table :easy_ml_features do |t|
+        t.bigint :dataset_id, null: false
+        t.string :name
+        t.bigint :version
+        t.string :feature_class, null: false
+        t.integer :feature_position
+        t.integer :batch_size
+        t.boolean :needs_fit
+        t.string :sha
+        t.string :primary_key, array: true
+        t.datetime :applied_at
+        t.datetime :fit_at
+        t.bigint :refresh_every
 
-      t.timestamps
+        t.timestamps
 
-      t.index %i[dataset_id feature_position], name: "idx_features_on_dataset_and_position"
-      t.index %i[dataset_id name], unique: true, name: "idx_features_on_dataset_and_name"
-      t.index :feature_class
-      t.index :applied_at
-      t.index :name
-      t.index :version
-      t.index :sha
-      t.index :batch_size
-      t.index :needs_fit
-      t.index :fit_at
-      t.index :refresh_every
+        t.index %i[dataset_id feature_position], name: "idx_features_on_dataset_and_position"
+        t.index %i[dataset_id name], unique: true, name: "idx_features_on_dataset_and_name"
+        t.index :feature_class
+        t.index :applied_at
+        t.index :name
+        t.index :version
+        t.index :sha
+        t.index :batch_size
+        t.index :needs_fit
+        t.index :fit_at
+        t.index :refresh_every
+      end
     end
   end
 end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_model_file_histories.rb.tt

@@ -2,8 +2,10 @@ require "historiographer/postgres_migration"
 
 class CreateEasyMLModelFileHistories < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
   def change
-    create_table :easy_ml_model_file_histories do |t|
-      t.histories(foreign_key: :model_file_id)
+    unless table_exists?(:easy_ml_model_file_histories)
+      create_table :easy_ml_model_file_histories do |t|
+        t.histories(foreign_key: :model_file_id)
+      end
     end
   end
-end 
+end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_model_files.rb.tt

@@ -1,17 +1,19 @@
 class CreateEasyMLModelFiles < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
   def change
-    create_table :easy_ml_model_files do |t|
-      t.string :filename, null: false
-      t.string :path, null: false
-      t.json :configuration
-      t.string :model_type
-      t.bigint :model_id
-      t.timestamps
+    unless table_exists?(:easy_ml_model_files)
+      create_table :easy_ml_model_files do |t|
+        t.string :filename, null: false
+        t.string :path, null: false
+        t.json :configuration
+        t.string :model_type
+        t.bigint :model_id
+        t.timestamps
 
-      t.index :created_at
-      t.index :filename
-      t.index [:model_type]
-      t.index :model_id
+        t.index :created_at
+        t.index :filename
+        t.index [:model_type]
+        t.index :model_id
+      end
     end
   end
 end

data/lib/easy_ml/railtie/templates/migration/create_easy_ml_model_histories.rb.tt

@@ -2,8 +2,10 @@ require "historiographer/postgres_migration"
 
 class CreateEasyMLModelHistories < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
   def change
-    create_table :easy_ml_model_histories do |t|
-      t.histories(foreign_key: :model_id)
+    unless table_exists?(:easy_ml_model_histories)
+      create_table :easy_ml_model_histories do |t|
+        t.histories(foreign_key: :model_id)
+      end
     end
   end
-end 
+end