online_migrations 0.27.1 → 0.29.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8a3658cc979ead6b8dea71140f3ba95b0f7a4a4c26a27891f49a5b6925c94e59
-  data.tar.gz: 7d3ab5e09422e28078543ac67fa0ecae1a7afe9d8e29f67e55d6b772fb708e50
+  metadata.gz: babb68ddc337cbc52cb3a10d842dfb427031544fa8b32716b9b0166996f45252
+  data.tar.gz: '08e334985d6ecbe603831ad44a9b535d3e39ee1b7f17bb576fc035b9de11c84c'
 SHA512:
-  metadata.gz: 43ae3f3ebffa5491d4ac9b99b987b048702b9142e9a4ae49182b705b9c81a562acbea6dc1580ef76e49a9124adf2d662723451b6d03003f10f05c1d39ff4a33e
-  data.tar.gz: 5793df9ffd71454f2edb251d079439c5dfb0cfe212a30360654cbb5622c23fcd47a63e2a8cdd6b8d4df2bd7d3b39ec4d29eede4a99519264075e65956bdd1b55
+  metadata.gz: ca383bfa73dd712076f3ac5ade02767955a7da7ec99306258d70a61b32331226fc22f4a84fea3f37423e7ebce2e805b514077bd13fcee1fcb09ae3ac637ac53f
+  data.tar.gz: 470a1d00e16f00b70a248c0ea2e0a3acfcb3e05b165fe5d12fa966c3f03d2a40c953fe4dd9265d210c330fca0b2536143716a47adfb9f1a8749d3348183e4959

data/CHANGELOG.md

@@ -1,5 +1,18 @@
 ## master (unreleased)
 
+## 0.29.0 (2025-07-10)
+
+- Add `iteration_pause` column to background data migrations
+
+    Note: Run `bin/rails generate online_migrations:upgrade` if using background data migrations.
+    Make sure all data migrations finished before applying this migration or change its timestamp
+    to be older than all pending db migrations which will enqueue data migrations.
+
+## 0.28.0 (2025-06-26)
+
+- Support renaming columns and tables for tables within custom schemas
+- Include operation name into the background schema migrations names
+
 ## 0.27.1 (2025-05-08)
 
 - Fix background data migrations to enumerate using the correct shard

data/lib/generators/online_migrations/templates/background_data_migrations_add_iteration_pause.rb.tt

@@ -0,0 +1,7 @@
+class BackgroundDataMigrationsAddIterationPause < <%= migration_parent %>
+  def change
+    safety_assured("Table is small") do
+      add_column :background_data_migrations, :iteration_pause, :float, default: 0.0, null: false
+    end
+  end
+end

data/lib/generators/online_migrations/templates/initializer.rb.tt

@@ -110,6 +110,9 @@ OnlineMigrations.configure do |config|
   # The number of seconds that must pass before the cancelling or pausing data migration is considered stuck.
   config.background_data_migrations.stuck_timeout = 5.minutes
 
+  # The pause interval between each data migration's `process` method execution (in seconds).
+  config.background_data_migrations.iteration_pause = Rails.env.production? ? 0.02 : 0
+
   # The callback to perform when an error occurs during the data migration.
   # config.background_data_migrations.error_handler = ->(error, errored_migration) do
   #   Bugsnag.notify(error) do |notification|

data/lib/generators/online_migrations/templates/install_migration.rb.tt

@@ -13,6 +13,7 @@ class InstallOnlineMigrations < <%= migration_parent %>
       t.bigint :tick_count, default: 0, null: false
       t.float :time_running, default: 0.0, null: false
       t.integer :max_attempts, null: false
+      t.float :iteration_pause, default: 0.0, null: false
       t.string :error_class
       t.string :error_message
       t.string :backtrace, array: true

data/lib/generators/online_migrations/upgrade_generator.rb

@@ -19,33 +19,34 @@ module OnlineMigrations
     private
       def migrations_to_apply
         connection = BackgroundDataMigrations::Migration.connection
-        data_table = "background_migrations"
-        schema_table = "background_schema_migrations"
-        columns = connection.columns(data_table).map(&:name)
 
         migrations = []
-        if connection.table_exists?(data_table) && !columns.include?("shard")
+        if connection.table_exists?(:background_migrations) && !connection.column_exists?(:background_migrations, :shard)
           migrations << "add_sharding_to_online_migrations"
         end
 
-        if !connection.table_exists?(schema_table)
+        if !connection.table_exists?(:background_schema_migrations)
           migrations << "create_background_schema_migrations"
         end
 
-        indexes = connection.indexes(schema_table)
+        indexes = connection.indexes(:background_schema_migrations)
         unique_index = indexes.find { |i| i.unique && i.columns.sort == ["connection_class_name", "migration_name", "shard"] }
         if !unique_index
           migrations << "background_schema_migrations_change_unique_index"
         end
 
-        if connection.table_exists?(data_table) && !connection.column_exists?(data_table, :started_at)
+        if connection.table_exists?(:background_migrations) && !connection.column_exists?(:background_migrations, :started_at)
           migrations << "add_timestamps_to_background_migrations"
         end
 
-        if connection.table_exists?(data_table)
+        if connection.table_exists?(:background_migrations)
           migrations << "change_background_data_migrations"
         end
 
+        if !connection.column_exists?(:background_data_migrations, :iteration_pause)
+          migrations << "background_data_migrations_add_iteration_pause"
+        end
+
         migrations
       end
 

data/lib/online_migrations/background_data_migrations/migration.rb

@@ -250,7 +250,7 @@ module OnlineMigrations
       def progress
         if succeeded?
           100.0
-        elsif enqueued? || tick_total == 0
+        elsif tick_total == 0
           0.0
         elsif tick_total
           ([tick_count.to_f / tick_total, 1.0].min * 100)
@@ -297,6 +297,8 @@ module OnlineMigrations
           self.tick_total ||= on_shard_if_present do
             data_migration.count
           end
+
+          self.iteration_pause ||= config.iteration_pause
         end
 
         def instrument_status_change

data/lib/online_migrations/background_data_migrations/migration_helpers.rb

@@ -369,7 +369,7 @@ module OnlineMigrations
       #     in development and test environments
       #
       def enqueue_background_data_migration(migration_name, *arguments, **options)
-        options.assert_valid_keys(:max_attempts, :connection_class_name)
+        options.assert_valid_keys(:max_attempts, :iteration_pause, :connection_class_name)
 
         migration_name = migration_name.name if migration_name.is_a?(Class)
         options[:connection_class_name] ||= compute_connection_class_name(migration_name, arguments)
@@ -387,7 +387,7 @@ module OnlineMigrations
           migration = Migration.where(migration_name: migration_name, shard: shard).where("arguments = ?", arguments.to_json).first
           migration ||= Migration.create!(**options, migration_name: migration_name, arguments: arguments, shard: shard)
 
-          if Utils.run_background_migrations_inline?
+          if Utils.run_background_migrations_inline? && !migration.succeeded?
             job = OnlineMigrations.config.background_data_migrations.job
             job.constantize.perform_inline(migration.id)
           end

data/lib/online_migrations/background_data_migrations/migration_job.rb

@@ -128,7 +128,8 @@ module OnlineMigrations
           @data_migration.around_process do
             @migration.data_migration.process(item)
 
-            pause = OnlineMigrations.config.background_data_migrations.iteration_pause
+            # Migration is refreshed regularly by ticker.
+            pause = @migration.iteration_pause
             sleep(pause) if pause > 0
           end
           @ticker.tick

data/lib/online_migrations/background_schema_migrations/migration_helpers.rb

@@ -28,7 +28,7 @@ module OnlineMigrations
         schema_creation = ActiveRecord::ConnectionAdapters::PostgreSQL::SchemaCreation.new(self)
         definition = schema_creation.accept(create_index)
 
-        enqueue_background_schema_migration(index.name, table_name, definition: definition, **migration_options)
+        enqueue_background_schema_migration("Add index #{index.name}", table_name, definition: definition, **migration_options)
       end
 
       def remove_index_in_background(table_name, column_name = nil, name:, **options)
@@ -42,7 +42,7 @@ module OnlineMigrations
         end
 
         definition = "DROP INDEX CONCURRENTLY IF EXISTS #{quote_column_name(name)}"
-        enqueue_background_schema_migration(name, table_name, definition: definition, **migration_options)
+        enqueue_background_schema_migration("Remove index #{name}", table_name, definition: definition, **migration_options)
       end
 
       def validate_foreign_key_in_background(from_table, to_table = nil, **options)
@@ -62,7 +62,7 @@ module OnlineMigrations
           ALTER TABLE #{quote_table_name(table_name)}
           VALIDATE CONSTRAINT #{quote_table_name(constraint_name)}
         SQL
-        enqueue_background_schema_migration(constraint_name, table_name, definition: definition, **options)
+        enqueue_background_schema_migration("Validate #{constraint_name}", table_name, definition: definition, **options)
       end
 
       # Ensures that the background schema migration with the provided migration name succeeded.
@@ -78,7 +78,7 @@ module OnlineMigrations
       #   ensure_background_schema_migration_succeeded("index_users_on_email")
       #
       def ensure_background_schema_migration_succeeded(migration_name)
-        migrations = Migration.where(migration_name: migration_name).to_a
+        migrations = Migration.where("migration_name ILIKE ?", "%#{migration_name}%").to_a
 
         if migrations.empty?
           Utils.raise_in_prod_or_say_in_dev("Could not find background schema migration(s): '#{migration_name}'.")
@@ -97,12 +97,13 @@ module OnlineMigrations
         if connection_class_name
           klass = connection_class_name.constantize
           connection_class = Utils.find_connection_class(klass)
-          # Normalize to the real connection class name.
-          connection_class_name = connection_class.name
         else
           connection_class = ActiveRecord::Base
         end
 
+        # Normalize to the real connection class name.
+        connection_class_name = connection_class.name
+
         shards = Utils.shard_names(connection_class)
         shards = [nil] if shards.size == 1
 
@@ -111,6 +112,9 @@ module OnlineMigrations
                                .find_or_create_by!(migration_name: migration_name, shard: shard, connection_class_name: connection_class_name)
 
           if Utils.run_background_migrations_inline?
+            # Run migration again in development.
+            migration.update_column(:status, :enqueued) if !migration.enqueued?
+
             runner = MigrationRunner.new(migration)
             runner.run
           end

data/lib/online_migrations/schema_cache.rb

@@ -52,15 +52,14 @@ module OnlineMigrations
     private
       def renamed_table?(connection, table_name)
         table_renames = OnlineMigrations.config.table_renames
-        if table_renames.key?(table_name)
-          views = connection.views
-          table_renames[table_name] if views.include?(table_name)
+        if table_renames.key?(table_name) && connection.view_exists?(table_name)
+          table_renames[table_name]
         end
       end
 
       def renamed_column?(connection, table_name)
         column_renames = OnlineMigrations.config.column_renames
-        column_renames.key?(table_name) && connection.views.include?(table_name)
+        column_renames.key?(table_name) && connection.view_exists?(table_name)
       end
 
       def column_rename_table(table_name)
@@ -131,14 +130,21 @@ module OnlineMigrations
       def renamed_table?(pool, table_name)
         table_renames = OnlineMigrations.config.table_renames
         if table_renames.key?(table_name)
-          views = pool.with_connection(&:views)
-          table_renames[table_name] if views.include?(table_name)
+          view_exists = pool.with_connection do |connection|
+            connection.view_exists?(table_name)
+          end
+
+          table_renames[table_name] if view_exists
         end
       end
 
       def renamed_column?(pool, table_name)
         column_renames = OnlineMigrations.config.column_renames
-        column_renames.key?(table_name) && pool.with_connection(&:views).include?(table_name)
+        return false if !column_renames.key?(table_name)
+
+        pool.with_connection do |connection|
+          connection.view_exists?(table_name)
+        end
       end
 
       def column_rename_table(table_name)

data/lib/online_migrations/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module OnlineMigrations
-  VERSION = "0.27.1"
+  VERSION = "0.29.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: online_migrations
 version: !ruby/object:Gem::Version
-  version: 0.27.1
+  version: 0.29.0
 platform: ruby
 authors:
 - fatkodima
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-05-08 00:00:00.000000000 Z
+date: 2025-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -42,6 +42,7 @@ files:
 - lib/generators/online_migrations/install_generator.rb
 - lib/generators/online_migrations/templates/add_sharding_to_online_migrations.rb.tt
 - lib/generators/online_migrations/templates/add_timestamps_to_background_migrations.rb.tt
+- lib/generators/online_migrations/templates/background_data_migrations_add_iteration_pause.rb.tt
 - lib/generators/online_migrations/templates/background_schema_migrations_change_unique_index.rb.tt
 - lib/generators/online_migrations/templates/change_background_data_migrations.rb.tt
 - lib/generators/online_migrations/templates/create_background_schema_migrations.rb.tt