online_migrations 0.26.0 → 0.27.1

data/lib/online_migrations/data_migration.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+gem "sidekiq", ">= 7.3.3"
+require "sidekiq"
+
+module OnlineMigrations
+  # Base class that is inherited by the host application's data migration classes.
+  class DataMigration
+    class NotFoundError < NameError; end
+
+    class << self
+      # Finds a Data Migration with the given name.
+      #
+      # @param name [String] the name of the Data Migration to be found.
+      #
+      # @return [DataMigration] the Data Migration with the given name.
+      #
+      # @raise [NotFoundError] if a Data Migration with the given name does not exist.
+      #
+      def named(name)
+        namespace = OnlineMigrations.config.background_data_migrations.migrations_module.constantize
+        internal_namespace = ::OnlineMigrations::BackgroundDataMigrations
+
+        migration = "#{namespace}::#{name}".safe_constantize ||
+                    "#{internal_namespace}::#{name}".safe_constantize
+
+        raise NotFoundError.new("Data Migration #{name} not found", name) if migration.nil?
+        if !(migration.is_a?(Class) && migration < self)
+          raise NotFoundError.new("#{name} is not a Data Migration", name)
+        end
+
+        migration
+      end
+
+      # @private
+      attr_accessor :active_record_enumerator_batch_size
+
+      # Limit the number of records that will be fetched in a single query when
+      # iterating over an Active Record collection migration.
+      #
+      # @param size [Integer] the number of records to fetch in a single query.
+      #
+      def collection_batch_size(size)
+        self.active_record_enumerator_batch_size = size
+      end
+    end
+
+    # A hook to override that will be called when the migration starts running.
+    #
+    def after_start
+    end
+
+    # A hook to override that will be called around 'process' each time.
+    #
+    # Can be useful for some metrics collection, performance tracking etc.
+    #
+    def around_process
+      yield
+    end
+
+    # A hook to override that will be called when the migration resumes its work.
+    #
+    def after_resume
+    end
+
+    # A hook to override that will be called each time the migration is interrupted.
+    #
+    # This can be due to interruption or sidekiq stopping.
+    #
+    def after_stop
+    end
+
+    # A hook to override that will be called when the migration finished its work.
+    #
+    def after_complete
+    end
+
+    # A hook to override that will be called when the migration is paused.
+    #
+    def after_pause
+    end
+
+    # A hook to override that will be called when the migration is cancelled.
+    #
+    def after_cancel
+    end
+
+    # The collection to be processed.
+    #
+    # @return [ActiveRecord::Relation, ActiveRecord::Batches::BatchEnumerator, Array, Enumerator]
+    #
+    # @raise [NotImplementedError] with a message advising subclasses to override this method.
+    #
+    def collection
+      raise NotImplementedError, "#{self.class.name} must implement a 'collection' method"
+    end
+
+    # The action to be performed on each item from the collection.
+    #
+    # @param _item the current item from the collection being iterated
+    # @raise [NotImplementedError] with a message advising subclasses to override this method.
+    #
+    def process(_item)
+      raise NotImplementedError, "#{self.class.name} must implement a 'process' method"
+    end
+
+    # Total count of iterations to be performed (optional, to be able to show progress).
+    #
+    # @return [Integer, nil]
+    #
+    def count
+    end
+
+    # Enumerator builder. You may override this method to return any Enumerator yielding
+    # pairs of `[item, item_cursor]`, instead of using `collection`.
+    #
+    # It is useful when it is not practical or impossible to define an explicit collection
+    # in the `collection` method.
+    #
+    # @param cursor [Object, nil] cursor position to resume from, or nil on initial call.
+    #
+    # @return [Enumerator]
+    #
+    def build_enumerator(cursor:)
+    end
+  end
+end

data/lib/online_migrations/error_messages.rb

@@ -230,6 +230,8 @@ during writes works automatically). For most column type changes, this does not
 7. Finally, if everything works as expected, remove copy trigger and old column:
 
   class Cleanup<%= migration_name %> < <%= migration_parent %>
+    disable_ddl_transaction!
+
     def up
       <%= cleanup_code %>
     end

data/lib/online_migrations/lock_retrier.rb

@@ -91,10 +91,13 @@ module OnlineMigrations
         else
           yield
         end
-      rescue ActiveRecord::LockWaitTimeout
+      rescue ActiveRecord::LockWaitTimeout, ActiveRecord::Deadlocked => e
         if current_attempt <= attempts
           current_delay = delay(current_attempt)
-          Utils.say("Lock timeout. Retrying in #{current_delay} seconds...")
+
+          problem = e.is_a?(ActiveRecord::Deadlocked) ? "Deadlock detected." : "Lock timeout."
+          Utils.say("#{problem} Retrying in #{current_delay} seconds...")
+
           sleep(current_delay)
           retry
         end

data/lib/online_migrations/schema_statements.rb

@@ -3,7 +3,7 @@
 module OnlineMigrations
   module SchemaStatements
     include ChangeColumnTypeHelpers
-    include BackgroundMigrations::MigrationHelpers
+    include BackgroundDataMigrations::MigrationHelpers
     include BackgroundSchemaMigrations::MigrationHelpers
 
     # Updates the value of a column in batches.

data/lib/online_migrations/shard_aware.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module OnlineMigrations
+  module ShardAware
+    extend ActiveSupport::Concern
+
+    included do
+      before_validation :set_connection_class_name
+    end
+
+    def on_shard_if_present(&block)
+      if shard
+        connection_class.connected_to(shard: shard.to_sym, role: :writing, &block)
+      else
+        yield
+      end
+    end
+
+    def connection_class_name=(value)
+      if value && (klass = value.safe_constantize)
+        if !(klass <= ActiveRecord::Base)
+          raise ArgumentError, "connection_class_name is not an ActiveRecord::Base child class"
+        end
+
+        connection_class = Utils.find_connection_class(klass)
+        super(connection_class.name)
+      end
+    end
+
+    # @private
+    def connection_class
+      if connection_class_name && (klass = connection_class_name.safe_constantize)
+        Utils.find_connection_class(klass)
+      else
+        ActiveRecord::Base
+      end
+    end
+
+    private
+      def set_connection_class_name
+        self.connection_class_name ||= "ActiveRecord::Base"
+      end
+  end
+end

data/lib/online_migrations/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module OnlineMigrations
-  VERSION = "0.26.0"
+  VERSION = "0.27.1"
 end

data/lib/online_migrations.rb

@@ -6,7 +6,8 @@ require "online_migrations/version"
 require "online_migrations/utils"
 require "online_migrations/background_schema_migrations/migration_helpers"
 require "online_migrations/change_column_type_helpers"
-require "online_migrations/background_migrations/migration_helpers"
+require "online_migrations/background_data_migrations/migration_helpers"
+require "online_migrations/active_record_batch_enumerator"
 require "online_migrations/schema_statements"
 require "online_migrations/schema_cache"
 require "online_migrations/migration"
@@ -29,7 +30,8 @@ module OnlineMigrations
   autoload :ForeignKeysCollector
   autoload :IndexDefinition
   autoload :CommandChecker
-  autoload :BackgroundMigration
+  autoload :DataMigration
+  autoload :ShardAware
 
   autoload_at "online_migrations/lock_retrier" do
     autoload :LockRetrier
@@ -40,24 +42,21 @@ module OnlineMigrations
 
   autoload :CopyTrigger
 
-  module BackgroundMigrations
+  module BackgroundDataMigrations
     extend ActiveSupport::Autoload
 
     autoload :Config
     autoload :MigrationStatusValidator
-    autoload :MigrationJobStatusValidator
-    autoload :BackgroundMigrationClassValidator
     autoload :BackfillColumn
     autoload :CopyColumn
     autoload :DeleteAssociatedRecords
     autoload :DeleteOrphanedRecords
     autoload :PerformActionOnRelation
     autoload :ResetCounters
-    autoload :MigrationJob
     autoload :Migration
-    autoload :MigrationJobRunner
-    autoload :MigrationRunner
+    autoload :MigrationJob
     autoload :Scheduler
+    autoload :Ticker
   end
 
   module BackgroundSchemaMigrations
@@ -70,6 +69,10 @@ module OnlineMigrations
     autoload :Scheduler
   end
 
+  # Make aliases for less typing.
+  DataMigrations = BackgroundDataMigrations
+  SchemaMigrations = BackgroundSchemaMigrations
+
   class << self
     # @private
     attr_accessor :current_migration
@@ -87,10 +90,10 @@ module OnlineMigrations
     # @option options [String, Symbol, nil] :shard The name of the shard to run
     #   background data migrations on. By default runs on all shards.
     #
-    def run_background_migrations(**options)
-      BackgroundMigrations::Scheduler.run(**options)
+    def run_background_data_migrations(**options)
+      BackgroundDataMigrations::Scheduler.run(**options)
     end
-    alias run_background_data_migrations run_background_migrations
+    alias run_background_migrations run_background_data_migrations
 
     # Run background schema migrations
     #
@@ -122,6 +125,10 @@ module OnlineMigrations
       else
         ActiveRecord::ConnectionAdapters::SchemaCache.prepend(OnlineMigrations::SchemaCache)
       end
+
+      if !ActiveRecord::Batches::BatchEnumerator.method_defined?(:use_ranges)
+        ActiveRecord::Batches::BatchEnumerator.include(OnlineMigrations::ActiveRecordBatchEnumerator)
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: online_migrations
 version: !ruby/object:Gem::Version
-  version: 0.26.0
+  version: 0.27.1
 platform: ruby
 authors:
 - fatkodima
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-04-27 00:00:00.000000000 Z
+date: 2025-05-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -34,39 +34,38 @@ files:
 - CHANGELOG.md
 - LICENSE.txt
 - README.md
+- docs/0.27-upgrade.md
 - docs/background_data_migrations.md
 - docs/background_schema_migrations.md
 - docs/configuring.md
-- lib/generators/online_migrations/background_migration_generator.rb
+- lib/generators/online_migrations/data_migration_generator.rb
 - lib/generators/online_migrations/install_generator.rb
 - lib/generators/online_migrations/templates/add_sharding_to_online_migrations.rb.tt
 - lib/generators/online_migrations/templates/add_timestamps_to_background_migrations.rb.tt
-- lib/generators/online_migrations/templates/background_data_migration.rb.tt
 - lib/generators/online_migrations/templates/background_schema_migrations_change_unique_index.rb.tt
+- lib/generators/online_migrations/templates/change_background_data_migrations.rb.tt
 - lib/generators/online_migrations/templates/create_background_schema_migrations.rb.tt
+- lib/generators/online_migrations/templates/data_migration.rb.tt
 - lib/generators/online_migrations/templates/initializer.rb.tt
 - lib/generators/online_migrations/templates/install_migration.rb.tt
 - lib/generators/online_migrations/templates/migration.rb.tt
 - lib/generators/online_migrations/upgrade_generator.rb
 - lib/online_migrations.rb
+- lib/online_migrations/active_record_batch_enumerator.rb
 - lib/online_migrations/application_record.rb
-- lib/online_migrations/background_migration.rb
-- lib/online_migrations/background_migrations/backfill_column.rb
-- lib/online_migrations/background_migrations/background_migration_class_validator.rb
-- lib/online_migrations/background_migrations/config.rb
-- lib/online_migrations/background_migrations/copy_column.rb
-- lib/online_migrations/background_migrations/delete_associated_records.rb
-- lib/online_migrations/background_migrations/delete_orphaned_records.rb
-- lib/online_migrations/background_migrations/migration.rb
-- lib/online_migrations/background_migrations/migration_helpers.rb
-- lib/online_migrations/background_migrations/migration_job.rb
-- lib/online_migrations/background_migrations/migration_job_runner.rb
-- lib/online_migrations/background_migrations/migration_job_status_validator.rb
-- lib/online_migrations/background_migrations/migration_runner.rb
-- lib/online_migrations/background_migrations/migration_status_validator.rb
-- lib/online_migrations/background_migrations/perform_action_on_relation.rb
-- lib/online_migrations/background_migrations/reset_counters.rb
-- lib/online_migrations/background_migrations/scheduler.rb
+- lib/online_migrations/background_data_migrations/backfill_column.rb
+- lib/online_migrations/background_data_migrations/config.rb
+- lib/online_migrations/background_data_migrations/copy_column.rb
+- lib/online_migrations/background_data_migrations/delete_associated_records.rb
+- lib/online_migrations/background_data_migrations/delete_orphaned_records.rb
+- lib/online_migrations/background_data_migrations/migration.rb
+- lib/online_migrations/background_data_migrations/migration_helpers.rb
+- lib/online_migrations/background_data_migrations/migration_job.rb
+- lib/online_migrations/background_data_migrations/migration_status_validator.rb
+- lib/online_migrations/background_data_migrations/perform_action_on_relation.rb
+- lib/online_migrations/background_data_migrations/reset_counters.rb
+- lib/online_migrations/background_data_migrations/scheduler.rb
+- lib/online_migrations/background_data_migrations/ticker.rb
 - lib/online_migrations/background_schema_migrations/config.rb
 - lib/online_migrations/background_schema_migrations/migration.rb
 - lib/online_migrations/background_schema_migrations/migration_helpers.rb
@@ -79,6 +78,7 @@ files:
 - lib/online_migrations/command_recorder.rb
 - lib/online_migrations/config.rb
 - lib/online_migrations/copy_trigger.rb
+- lib/online_migrations/data_migration.rb
 - lib/online_migrations/database_tasks.rb
 - lib/online_migrations/error_messages.rb
 - lib/online_migrations/foreign_keys_collector.rb
@@ -89,6 +89,7 @@ files:
 - lib/online_migrations/schema_cache.rb
 - lib/online_migrations/schema_dumper.rb
 - lib/online_migrations/schema_statements.rb
+- lib/online_migrations/shard_aware.rb
 - lib/online_migrations/utils.rb
 - lib/online_migrations/verbose_sql_logs.rb
 - lib/online_migrations/version.rb

data/lib/online_migrations/background_migration.rb

@@ -1,64 +0,0 @@
-# frozen_string_literal: true
-
-module OnlineMigrations
-  # Base class that is inherited by the host application's background migration classes.
-  class BackgroundMigration
-    class NotFoundError < NameError; end
-
-    class << self
-      # Finds a Background Migration with the given name.
-      #
-      # @param name [String] the name of the Background Migration to be found.
-      #
-      # @return [BackgroundMigration] the Background Migration with the given name.
-      #
-      # @raise [NotFoundError] if a Background Migration with the given name does not exist.
-      #
-      def named(name)
-        namespace = OnlineMigrations.config.background_migrations.migrations_module.constantize
-        internal_namespace = ::OnlineMigrations::BackgroundMigrations
-
-        migration = "#{namespace}::#{name}".safe_constantize ||
-                    "#{internal_namespace}::#{name}".safe_constantize
-
-        raise NotFoundError.new("Background Migration #{name} not found", name) if migration.nil?
-        if !(migration.is_a?(Class) && migration < self)
-          raise NotFoundError.new("#{name} is not a Background Migration", name)
-        end
-
-        migration
-      end
-    end
-
-    # The relation to be iterated over.
-    #
-    # @return [ActiveRecord::Relation]
-    #
-    # @raise [NotImplementedError] with a message advising subclasses to
-    #     implement an override for this method.
-    #
-    def relation
-      raise NotImplementedError, "#{self.class.name} must implement a 'relation' method"
-    end
-
-    # Processes one batch.
-    #
-    # @param _relation [ActiveRecord::Relation] the current batch from the enumerator being iterated
-    # @return [void]
-    #
-    # @raise [NotImplementedError] with a message advising subclasses to
-    #     implement an override for this method.
-    #
-    def process_batch(_relation)
-      raise NotImplementedError, "#{self.class.name} must implement a 'process_batch' method"
-    end
-
-    # Returns the count of rows that will be iterated over (optional, to be able to show progress).
-    #
-    # @return [Integer, nil, :no_count]
-    #
-    def count
-      :no_count
-    end
-  end
-end

data/lib/online_migrations/background_migrations/backfill_column.rb

@@ -1,54 +0,0 @@
-# frozen_string_literal: true
-
-module OnlineMigrations
-  module BackgroundMigrations
-    # @private
-    class BackfillColumn < BackgroundMigration
-      attr_reader :table_name, :updates, :model_name
-
-      def initialize(table_name, updates, model_name = nil)
-        @table_name = table_name
-        @updates = updates
-        @model_name = model_name
-      end
-
-      def relation
-        column, value = updates.first
-
-        if updates.size == 1 && !value.nil?
-          # If value is nil, the generated SQL is correct (`WHERE column IS NOT NULL`).
-          # Otherwise, the SQL is `WHERE column != value`. This condition ignores column
-          # with NULLs in it, so we need to also manually check for NULLs.
-          quoted_column = connection.quote_column_name(column)
-          model.unscoped.where("#{quoted_column} != ? OR #{quoted_column} IS NULL", value)
-        else
-          model.unscoped.where.not(updates)
-        end
-      end
-
-      def process_batch(relation)
-        relation.update_all(updates)
-      end
-
-      def count
-        # Exact counts are expensive on large tables, since PostgreSQL
-        # needs to do a full scan. An estimated count should give a pretty decent
-        # approximation of rows count in this case.
-        Utils.estimated_count(connection, table_name)
-      end
-
-      private
-        def model
-          @model ||= if model_name.present?
-                       Object.const_get(model_name, false)
-                     else
-                       Utils.define_model(table_name)
-                     end
-        end
-
-        def connection
-          model.connection
-        end
-    end
-  end
-end

data/lib/online_migrations/background_migrations/background_migration_class_validator.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module OnlineMigrations
-  module BackgroundMigrations
-    # @private
-    class BackgroundMigrationClassValidator < ActiveModel::Validator
-      def validate(record)
-        relation = record.migration_relation
-        migration_name = record.migration_name
-
-        if !relation.is_a?(ActiveRecord::Relation)
-          record.errors.add(
-            :migration_name,
-            "#{migration_name}#relation must return an ActiveRecord::Relation object"
-          )
-          return
-        end
-
-        if relation.arel.orders.present? || relation.arel.taken.present?
-          record.errors.add(
-            :migration_name,
-            "#{migration_name}#relation cannot use ORDER BY or LIMIT due to the way how iteration with a cursor is designed. " \
-            "You can use other ways to limit the number of rows, e.g. a WHERE condition on the primary key column."
-          )
-        end
-      end
-    end
-  end
-end

data/lib/online_migrations/background_migrations/config.rb

@@ -1,74 +0,0 @@
-# frozen_string_literal: true
-
-module OnlineMigrations
-  module BackgroundMigrations
-    # Class representing configuration options for background migrations.
-    class Config
-      # The path where generated background migrations will be placed
-      # @return [String] defaults to "lib"
-      attr_accessor :migrations_path
-
-      # The module in which background migrations will be placed
-      # @return [String] defaults to "OnlineMigrations::BackgroundMigrations"
-      attr_accessor :migrations_module
-
-      # The number of rows to process in a single background migration run
-      # @return [Integer] defaults to 1_000
-      #
-      attr_accessor :batch_size
-
-      # The smaller batches size that the batches will be divided into
-      # @return [Integer] defaults to 100
-      #
-      attr_accessor :sub_batch_size
-
-      # The pause interval between each background migration job's execution (in seconds)
-      # @return [Integer] defaults to 0
-      #
-      attr_accessor :batch_pause
-
-      # The number of milliseconds to sleep between each sub_batch execution
-      # @return [Integer] defaults to 100 milliseconds
-      #
-      attr_accessor :sub_batch_pause_ms
-
-      # Maximum number of batch run attempts
-      #
-      # When attempts are exhausted, the individual batch is marked as failed.
-      # @return [Integer] defaults to 5
-      #
-      attr_accessor :batch_max_attempts
-
-      # The number of seconds that must pass before the running job is considered stuck
-      #
-      # @return [Integer] defaults to 1 hour
-      #
-      attr_accessor :stuck_jobs_timeout
-
-      # The callback to perform when an error occurs in the migration job.
-      #
-      # @example
-      #   OnlineMigrations.config.background_migrations.error_handler = ->(error, errored_job) do
-      #     Bugsnag.notify(error) do |notification|
-      #       notification.add_metadata(:background_migration, { name: errored_job.migration_name })
-      #     end
-      #   end
-      #
-      # @return [Proc] the callback to perform when an error occurs in the migration job
-      #
-      attr_accessor :error_handler
-
-      def initialize
-        @migrations_path = "lib"
-        @migrations_module = "OnlineMigrations::BackgroundMigrations"
-        @batch_size = 1_000
-        @sub_batch_size = 100
-        @batch_pause = 0.seconds
-        @sub_batch_pause_ms = 100
-        @batch_max_attempts = 5
-        @stuck_jobs_timeout = 1.hour
-        @error_handler = ->(error, errored_job) {}
-      end
-    end
-  end
-end