online_migrations 0.26.0 → 0.27.0

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: online_migrations
 version: !ruby/object:Gem::Version
-  version: 0.26.0
+  version: 0.27.0
 platform: ruby
 authors:
 - fatkodima
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-04-27 00:00:00.000000000 Z
+date: 2025-05-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -34,39 +34,38 @@ files:
 - CHANGELOG.md
 - LICENSE.txt
 - README.md
+- docs/0.27-upgrade.md
 - docs/background_data_migrations.md
 - docs/background_schema_migrations.md
 - docs/configuring.md
-- lib/generators/online_migrations/background_migration_generator.rb
+- lib/generators/online_migrations/data_migration_generator.rb
 - lib/generators/online_migrations/install_generator.rb
 - lib/generators/online_migrations/templates/add_sharding_to_online_migrations.rb.tt
 - lib/generators/online_migrations/templates/add_timestamps_to_background_migrations.rb.tt
-- lib/generators/online_migrations/templates/background_data_migration.rb.tt
 - lib/generators/online_migrations/templates/background_schema_migrations_change_unique_index.rb.tt
+- lib/generators/online_migrations/templates/change_background_data_migrations.rb.tt
 - lib/generators/online_migrations/templates/create_background_schema_migrations.rb.tt
+- lib/generators/online_migrations/templates/data_migration.rb.tt
 - lib/generators/online_migrations/templates/initializer.rb.tt
 - lib/generators/online_migrations/templates/install_migration.rb.tt
 - lib/generators/online_migrations/templates/migration.rb.tt
 - lib/generators/online_migrations/upgrade_generator.rb
 - lib/online_migrations.rb
+- lib/online_migrations/active_record_batch_enumerator.rb
 - lib/online_migrations/application_record.rb
-- lib/online_migrations/background_migration.rb
-- lib/online_migrations/background_migrations/backfill_column.rb
-- lib/online_migrations/background_migrations/background_migration_class_validator.rb
-- lib/online_migrations/background_migrations/config.rb
-- lib/online_migrations/background_migrations/copy_column.rb
-- lib/online_migrations/background_migrations/delete_associated_records.rb
-- lib/online_migrations/background_migrations/delete_orphaned_records.rb
-- lib/online_migrations/background_migrations/migration.rb
-- lib/online_migrations/background_migrations/migration_helpers.rb
-- lib/online_migrations/background_migrations/migration_job.rb
-- lib/online_migrations/background_migrations/migration_job_runner.rb
-- lib/online_migrations/background_migrations/migration_job_status_validator.rb
-- lib/online_migrations/background_migrations/migration_runner.rb
-- lib/online_migrations/background_migrations/migration_status_validator.rb
-- lib/online_migrations/background_migrations/perform_action_on_relation.rb
-- lib/online_migrations/background_migrations/reset_counters.rb
-- lib/online_migrations/background_migrations/scheduler.rb
+- lib/online_migrations/background_data_migrations/backfill_column.rb
+- lib/online_migrations/background_data_migrations/config.rb
+- lib/online_migrations/background_data_migrations/copy_column.rb
+- lib/online_migrations/background_data_migrations/delete_associated_records.rb
+- lib/online_migrations/background_data_migrations/delete_orphaned_records.rb
+- lib/online_migrations/background_data_migrations/migration.rb
+- lib/online_migrations/background_data_migrations/migration_helpers.rb
+- lib/online_migrations/background_data_migrations/migration_job.rb
+- lib/online_migrations/background_data_migrations/migration_status_validator.rb
+- lib/online_migrations/background_data_migrations/perform_action_on_relation.rb
+- lib/online_migrations/background_data_migrations/reset_counters.rb
+- lib/online_migrations/background_data_migrations/scheduler.rb
+- lib/online_migrations/background_data_migrations/ticker.rb
 - lib/online_migrations/background_schema_migrations/config.rb
 - lib/online_migrations/background_schema_migrations/migration.rb
 - lib/online_migrations/background_schema_migrations/migration_helpers.rb
@@ -79,6 +78,7 @@ files:
 - lib/online_migrations/command_recorder.rb
 - lib/online_migrations/config.rb
 - lib/online_migrations/copy_trigger.rb
+- lib/online_migrations/data_migration.rb
 - lib/online_migrations/database_tasks.rb
 - lib/online_migrations/error_messages.rb
 - lib/online_migrations/foreign_keys_collector.rb
@@ -89,6 +89,7 @@ files:
 - lib/online_migrations/schema_cache.rb
 - lib/online_migrations/schema_dumper.rb
 - lib/online_migrations/schema_statements.rb
+- lib/online_migrations/shard_aware.rb
 - lib/online_migrations/utils.rb
 - lib/online_migrations/verbose_sql_logs.rb
 - lib/online_migrations/version.rb

data/lib/online_migrations/background_migration.rb

@@ -1,64 +0,0 @@
-# frozen_string_literal: true
-
-module OnlineMigrations
-  # Base class that is inherited by the host application's background migration classes.
-  class BackgroundMigration
-    class NotFoundError < NameError; end
-
-    class << self
-      # Finds a Background Migration with the given name.
-      #
-      # @param name [String] the name of the Background Migration to be found.
-      #
-      # @return [BackgroundMigration] the Background Migration with the given name.
-      #
-      # @raise [NotFoundError] if a Background Migration with the given name does not exist.
-      #
-      def named(name)
-        namespace = OnlineMigrations.config.background_migrations.migrations_module.constantize
-        internal_namespace = ::OnlineMigrations::BackgroundMigrations
-
-        migration = "#{namespace}::#{name}".safe_constantize ||
-                    "#{internal_namespace}::#{name}".safe_constantize
-
-        raise NotFoundError.new("Background Migration #{name} not found", name) if migration.nil?
-        if !(migration.is_a?(Class) && migration < self)
-          raise NotFoundError.new("#{name} is not a Background Migration", name)
-        end
-
-        migration
-      end
-    end
-
-    # The relation to be iterated over.
-    #
-    # @return [ActiveRecord::Relation]
-    #
-    # @raise [NotImplementedError] with a message advising subclasses to
-    #     implement an override for this method.
-    #
-    def relation
-      raise NotImplementedError, "#{self.class.name} must implement a 'relation' method"
-    end
-
-    # Processes one batch.
-    #
-    # @param _relation [ActiveRecord::Relation] the current batch from the enumerator being iterated
-    # @return [void]
-    #
-    # @raise [NotImplementedError] with a message advising subclasses to
-    #     implement an override for this method.
-    #
-    def process_batch(_relation)
-      raise NotImplementedError, "#{self.class.name} must implement a 'process_batch' method"
-    end
-
-    # Returns the count of rows that will be iterated over (optional, to be able to show progress).
-    #
-    # @return [Integer, nil, :no_count]
-    #
-    def count
-      :no_count
-    end
-  end
-end

data/lib/online_migrations/background_migrations/backfill_column.rb

@@ -1,54 +0,0 @@
-# frozen_string_literal: true
-
-module OnlineMigrations
-  module BackgroundMigrations
-    # @private
-    class BackfillColumn < BackgroundMigration
-      attr_reader :table_name, :updates, :model_name
-
-      def initialize(table_name, updates, model_name = nil)
-        @table_name = table_name
-        @updates = updates
-        @model_name = model_name
-      end
-
-      def relation
-        column, value = updates.first
-
-        if updates.size == 1 && !value.nil?
-          # If value is nil, the generated SQL is correct (`WHERE column IS NOT NULL`).
-          # Otherwise, the SQL is `WHERE column != value`. This condition ignores column
-          # with NULLs in it, so we need to also manually check for NULLs.
-          quoted_column = connection.quote_column_name(column)
-          model.unscoped.where("#{quoted_column} != ? OR #{quoted_column} IS NULL", value)
-        else
-          model.unscoped.where.not(updates)
-        end
-      end
-
-      def process_batch(relation)
-        relation.update_all(updates)
-      end
-
-      def count
-        # Exact counts are expensive on large tables, since PostgreSQL
-        # needs to do a full scan. An estimated count should give a pretty decent
-        # approximation of rows count in this case.
-        Utils.estimated_count(connection, table_name)
-      end
-
-      private
-        def model
-          @model ||= if model_name.present?
-                       Object.const_get(model_name, false)
-                     else
-                       Utils.define_model(table_name)
-                     end
-        end
-
-        def connection
-          model.connection
-        end
-    end
-  end
-end

data/lib/online_migrations/background_migrations/background_migration_class_validator.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module OnlineMigrations
-  module BackgroundMigrations
-    # @private
-    class BackgroundMigrationClassValidator < ActiveModel::Validator
-      def validate(record)
-        relation = record.migration_relation
-        migration_name = record.migration_name
-
-        if !relation.is_a?(ActiveRecord::Relation)
-          record.errors.add(
-            :migration_name,
-            "#{migration_name}#relation must return an ActiveRecord::Relation object"
-          )
-          return
-        end
-
-        if relation.arel.orders.present? || relation.arel.taken.present?
-          record.errors.add(
-            :migration_name,
-            "#{migration_name}#relation cannot use ORDER BY or LIMIT due to the way how iteration with a cursor is designed. " \
-            "You can use other ways to limit the number of rows, e.g. a WHERE condition on the primary key column."
-          )
-        end
-      end
-    end
-  end
-end

data/lib/online_migrations/background_migrations/config.rb

@@ -1,74 +0,0 @@
-# frozen_string_literal: true
-
-module OnlineMigrations
-  module BackgroundMigrations
-    # Class representing configuration options for background migrations.
-    class Config
-      # The path where generated background migrations will be placed
-      # @return [String] defaults to "lib"
-      attr_accessor :migrations_path
-
-      # The module in which background migrations will be placed
-      # @return [String] defaults to "OnlineMigrations::BackgroundMigrations"
-      attr_accessor :migrations_module
-
-      # The number of rows to process in a single background migration run
-      # @return [Integer] defaults to 1_000
-      #
-      attr_accessor :batch_size
-
-      # The smaller batches size that the batches will be divided into
-      # @return [Integer] defaults to 100
-      #
-      attr_accessor :sub_batch_size
-
-      # The pause interval between each background migration job's execution (in seconds)
-      # @return [Integer] defaults to 0
-      #
-      attr_accessor :batch_pause
-
-      # The number of milliseconds to sleep between each sub_batch execution
-      # @return [Integer] defaults to 100 milliseconds
-      #
-      attr_accessor :sub_batch_pause_ms
-
-      # Maximum number of batch run attempts
-      #
-      # When attempts are exhausted, the individual batch is marked as failed.
-      # @return [Integer] defaults to 5
-      #
-      attr_accessor :batch_max_attempts
-
-      # The number of seconds that must pass before the running job is considered stuck
-      #
-      # @return [Integer] defaults to 1 hour
-      #
-      attr_accessor :stuck_jobs_timeout
-
-      # The callback to perform when an error occurs in the migration job.
-      #
-      # @example
-      #   OnlineMigrations.config.background_migrations.error_handler = ->(error, errored_job) do
-      #     Bugsnag.notify(error) do |notification|
-      #       notification.add_metadata(:background_migration, { name: errored_job.migration_name })
-      #     end
-      #   end
-      #
-      # @return [Proc] the callback to perform when an error occurs in the migration job
-      #
-      attr_accessor :error_handler
-
-      def initialize
-        @migrations_path = "lib"
-        @migrations_module = "OnlineMigrations::BackgroundMigrations"
-        @batch_size = 1_000
-        @sub_batch_size = 100
-        @batch_pause = 0.seconds
-        @sub_batch_pause_ms = 100
-        @batch_max_attempts = 5
-        @stuck_jobs_timeout = 1.hour
-        @error_handler = ->(error, errored_job) {}
-      end
-    end
-  end
-end

data/lib/online_migrations/background_migrations/migration.rb

@@ -1,329 +0,0 @@
-# frozen_string_literal: true
-
-module OnlineMigrations
-  module BackgroundMigrations
-    # Class representing background data migration.
-    #
-    # @note The records of this class should not be created manually, but via
-    #   `enqueue_background_data_migration` helper inside migrations.
-    #
-    class Migration < ApplicationRecord
-      STATUSES = [
-        :enqueued,    # The migration has been enqueued by the user.
-        :running,     # The migration is being performed by a migration executor.
-        :paused,      # The migration was paused in the middle of the run by the user.
-        :finishing,   # The migration is being manually finishing inline by the user.
-        :failed,      # The migration raises an exception when running.
-        :succeeded,   # The migration finished without error.
-        :cancelled,   # The migration was cancelled by the user.
-      ]
-
-      self.table_name = :background_migrations
-
-      scope :queue_order, -> { order(created_at: :asc) }
-      scope :parents, -> { where(parent_id: nil) }
-      scope :runnable, -> { where(composite: false) }
-      scope :active, -> { where(status: [statuses[:enqueued], statuses[:running]]) }
-      scope :except_succeeded, -> { where.not(status: :succeeded) }
-      scope :for_migration_name, ->(migration_name) { where(migration_name: normalize_migration_name(migration_name)) }
-      scope :for_configuration, ->(migration_name, arguments) do
-        for_migration_name(migration_name).where("arguments = ?", arguments.to_json)
-      end
-
-      alias_attribute :name, :migration_name
-
-      enum :status, STATUSES.index_with(&:to_s)
-
-      belongs_to :parent, class_name: name, optional: true, inverse_of: :children
-      has_many :children, class_name: name, foreign_key: :parent_id, dependent: :delete_all, inverse_of: :parent
-      has_many :migration_jobs, dependent: :delete_all, inverse_of: :migration
-
-      validates :migration_name, :batch_column_name, presence: true
-
-      validates :batch_size, :sub_batch_size, presence: true, numericality: { greater_than: 0 }
-      validates :min_value, :max_value, presence: true, numericality: { greater_than: 0, unless: :composite? }
-
-      validates :batch_pause, :sub_batch_pause_ms, presence: true,
-                  numericality: { greater_than_or_equal_to: 0 }
-      validates :rows_count, numericality: { greater_than_or_equal_to: 0 }, allow_nil: true, unless: :composite?
-      validates :arguments, uniqueness: { scope: [:migration_name, :shard] }
-
-      validate :validate_batch_column_values
-      validate :validate_batch_sizes
-      validate :validate_jobs_status, if: :status_changed?
-
-      validates_with BackgroundMigrationClassValidator
-      validates_with MigrationStatusValidator, on: :update
-
-      before_validation :set_defaults
-      before_update :copy_attributes_to_children, if: :composite?
-
-      # @private
-      def self.normalize_migration_name(migration_name)
-        namespace = ::OnlineMigrations.config.background_migrations.migrations_module
-        migration_name.sub(/^(::)?#{namespace}::/, "")
-      end
-
-      def migration_name=(class_name)
-        class_name = class_name.name if class_name.is_a?(Class)
-        write_attribute(:migration_name, self.class.normalize_migration_name(class_name))
-      end
-      alias name= migration_name=
-
-      def completed?
-        succeeded? || failed?
-      end
-
-      # Overwrite enum's generated method to correctly work for composite migrations.
-      def paused!
-        return super if !composite?
-
-        transaction do
-          super
-          children.each { |child| child.paused! if child.enqueued? || child.running? }
-        end
-      end
-
-      # Overwrite enum's generated method to correctly work for composite migrations.
-      def running!
-        return super if !composite?
-
-        transaction do
-          super
-          children.each { |child| child.running! if child.paused? }
-        end
-      end
-
-      # Overwrite enum's generated method to correctly work for composite migrations.
-      def cancelled!
-        return super if !composite?
-
-        transaction do
-          super
-          children.each { |child| child.cancelled! if !child.succeeded? }
-        end
-      end
-      alias cancel cancelled!
-
-      def pausable?
-        true
-      end
-
-      def can_be_paused?
-        enqueued? || running?
-      end
-
-      def can_be_cancelled?
-        !succeeded? && !cancelled?
-      end
-
-      def last_job
-        migration_jobs.order(:max_value).last
-      end
-
-      # Returns the progress of the background migration.
-      #
-      # @return [Float, nil]
-      #   - when background migration is configured to not track progress, returns `nil`
-      #   - otherwise returns value in range from 0.0 to 100.0
-      #
-      def progress
-        if succeeded?
-          100.0
-        elsif enqueued?
-          0.0
-        elsif composite?
-          rows_counts = children.to_a.pluck(:rows_count)
-          if rows_counts.none?(nil)
-            total_rows_count = rows_counts.sum
-            return 100.0 if total_rows_count == 0
-
-            progresses = children.map do |child|
-              child.progress * child.rows_count / total_rows_count # weighted progress
-            end
-
-            progresses.sum.round(2)
-          end
-        elsif rows_count
-          if rows_count > 0 && rows_count > batch_size
-            jobs_rows_count = migration_jobs.succeeded.sum(:batch_size)
-            # The last migration job may need to process the amount of rows
-            # less than the batch size, so we can get a value > 1.0.
-            ([jobs_rows_count.to_f / rows_count, 1.0].min * 100).round(2)
-          else
-            0.0
-          end
-        end
-      end
-
-      def migration_class
-        BackgroundMigration.named(migration_name)
-      end
-
-      def migration_object
-        @migration_object ||= migration_class.new(*arguments)
-      end
-
-      def migration_relation
-        migration_object.relation
-      end
-
-      def migration_model
-        migration_relation.model
-      end
-
-      # Returns whether the interval between previous step run has passed.
-      # @return [Boolean]
-      #
-      def interval_elapsed?
-        last_job = migration_jobs.order(:updated_at).last
-        return true if last_job.nil?
-
-        last_job.enqueued? || (last_job.updated_at + batch_pause <= Time.current)
-      end
-
-      # Mark this migration as ready to be processed again.
-      #
-      # This method marks failed jobs as ready to be processed again, and
-      # they will be picked up on the next Scheduler run.
-      #
-      def retry
-        if composite? && failed?
-          transaction do
-            update!(status: :enqueued, finished_at: nil)
-            children.failed.each(&:retry)
-          end
-
-          true
-        elsif failed?
-          transaction do
-            parent.update!(status: :enqueued, finished_at: nil) if parent
-            update!(status: :enqueued, started_at: nil, finished_at: nil)
-
-            iterator = BatchIterator.new(migration_jobs.failed)
-            iterator.each_batch(of: 100) do |batch|
-              batch.each(&:retry)
-            end
-          end
-
-          true
-        else
-          false
-        end
-      end
-      alias retry_failed_jobs retry
-
-      # @private
-      def on_shard(&block)
-        abstract_class = Utils.find_connection_class(migration_model)
-
-        shard = (self.shard || abstract_class.default_shard).to_sym
-        abstract_class.connected_to(shard: shard, role: :writing, &block)
-      end
-
-      # @private
-      def reset_failed_jobs_attempts
-        iterator = BatchIterator.new(migration_jobs.failed)
-        iterator.each_batch(of: 100) do |relation|
-          relation.update_all(status: :enqueued, attempts: 0)
-        end
-      end
-
-      # @private
-      def next_batch_range
-        iterator = BatchIterator.new(migration_relation)
-        batch_range = nil
-
-        on_shard do
-          # rubocop:disable Lint/UnreachableLoop
-          iterator.each_batch(of: batch_size, column: batch_column_name, start: next_min_value, finish: max_value) do |_relation, min_value, max_value|
-            batch_range = [min_value, max_value]
-
-            break
-          end
-          # rubocop:enable Lint/UnreachableLoop
-        end
-
-        return if batch_range.nil?
-
-        min_value, max_value = batch_range
-        return if min_value > self.max_value
-
-        max_value = [max_value, self.max_value].min
-
-        [min_value, max_value]
-      end
-
-      private
-        def validate_batch_column_values
-          if max_value.to_i < min_value.to_i
-            errors.add(:base, "max_value should be greater than or equal to min_value")
-          end
-        end
-
-        def validate_batch_sizes
-          if sub_batch_size.to_i > batch_size.to_i
-            errors.add(:base, "sub_batch_size should be smaller than or equal to batch_size")
-          end
-        end
-
-        def validate_jobs_status
-          if composite?
-            if succeeded? && children.except_succeeded.exists?
-              errors.add(:base, "all child migrations must be succeeded")
-            elsif failed? && !children.failed.exists?
-              errors.add(:base, "at least one child migration must be failed")
-            end
-          elsif succeeded? && migration_jobs.except_succeeded.exists?
-            errors.add(:base, "all migration jobs must be succeeded")
-          elsif failed? && !migration_jobs.failed.exists?
-            errors.add(:base, "at least one migration job must be failed")
-          end
-        end
-
-        def set_defaults
-          if migration_relation.is_a?(ActiveRecord::Relation)
-            self.batch_column_name ||= migration_relation.primary_key
-
-            if composite?
-              self.min_value = self.max_value = self.rows_count = -1 # not relevant
-            else
-              on_shard do
-                # Getting exact min/max values can be a very heavy operation
-                # and is not needed practically.
-                self.min_value ||= 1
-                self.max_value ||= migration_model.unscoped.maximum(batch_column_name) || self.min_value
-
-                count = migration_object.count
-                self.rows_count = count if count != :no_count
-              end
-            end
-          end
-
-          config = ::OnlineMigrations.config.background_migrations
-          self.batch_size           ||= config.batch_size
-          self.sub_batch_size       ||= config.sub_batch_size
-          self.batch_pause          ||= config.batch_pause
-          self.sub_batch_pause_ms   ||= config.sub_batch_pause_ms
-          self.batch_max_attempts   ||= config.batch_max_attempts
-        end
-
-        def copy_attributes_to_children
-          attributes = [:batch_size, :sub_batch_size, :batch_pause, :sub_batch_pause_ms, :batch_max_attempts]
-          updates = {}
-          attributes.each do |attribute|
-            updates[attribute] = read_attribute(attribute) if attribute_changed?(attribute)
-          end
-          children.active.update_all(updates) if updates.any?
-        end
-
-        def next_min_value
-          if last_job
-            last_job.max_value.next
-          else
-            min_value
-          end
-        end
-    end
-  end
-end