online_migrations 0.25.0 → 0.27.0

data/lib/online_migrations/schema_statements.rb

@@ -3,7 +3,7 @@
 module OnlineMigrations
   module SchemaStatements
     include ChangeColumnTypeHelpers
-    include BackgroundMigrations::MigrationHelpers
+    include BackgroundDataMigrations::MigrationHelpers
     include BackgroundSchemaMigrations::MigrationHelpers
 
     # Updates the value of a column in batches.
@@ -568,8 +568,8 @@ module OnlineMigrations
     #
     def add_text_limit_constraint(table_name, column_name, limit, name: nil, validate: true)
       column = column_for(table_name, column_name)
-      if column.type != :text
-        raise "add_text_limit_constraint must be used only with :text columns"
+      if column.type != :text && column.type != :string
+        raise "add_text_limit_constraint must be used only with :text or :string columns"
       end
 
       name ||= __text_limit_constraint_name(table_name, column_name)
@@ -710,16 +710,12 @@ module OnlineMigrations
           index_name = (options[:name] || index_name(table_name, column_name)).to_s
           indexes(table_name).find { |i| i.name == index_name }
         else
-          # Rewrite this with `IndexDefinition#defined_for?` when Active Record >= 7.1 is supported.
-          # See https://github.com/rails/rails/pull/45160.
-          indexes(table_name).find { |i| __index_defined_for?(i, column_name, **options) }
+          indexes(table_name).find { |i| i.defined_for?(column_name, **options) }
         end
 
       if index
-        schema = __schema_for_table(table_name)
-
-        if __index_valid?(index.name, schema: schema)
-          Utils.say("Index was not created because it already exists.")
+        if index.valid?
+          Utils.say("Index #{index.name} was not created because it already exists.")
           return
         else
           Utils.say("Recreating invalid index: table_name: #{table_name}, column_name: #{column_name}")
@@ -764,22 +760,6 @@ module OnlineMigrations
       end
     end
 
-    # @private
-    # From ActiveRecord. Will not be needed for ActiveRecord >= 7.1.
-    def index_name(table_name, options)
-      if options.is_a?(Hash)
-        if options[:column]
-          Utils.index_name(table_name, options[:column])
-        elsif options[:name]
-          options[:name]
-        else
-          raise ArgumentError, "You must specify the index name"
-        end
-      else
-        index_name(table_name, column: options)
-      end
-    end
-
     # Extends default method to be idempotent.
     #
     # @see https://edgeapi.rubyonrails.org/classes/ActiveRecord/ConnectionAdapters/SchemaStatements.html#method-i-add_foreign_key
@@ -833,7 +813,7 @@ module OnlineMigrations
     # @see https://edgeapi.rubyonrails.org/classes/ActiveRecord/ConnectionAdapters/SchemaStatements.html#method-i-add_check_constraint
     #
     def add_check_constraint(table_name, expression, **options)
-      if __check_constraint_exists?(table_name, expression: expression, **options)
+      if check_constraint_exists?(table_name, expression: expression, **options)
         Utils.say(<<~MSG.squish)
           Check constraint was not created because it already exists (this may be due to an aborted migration or similar).
           table_name: #{table_name}, expression: #{expression}
@@ -864,7 +844,7 @@ module OnlineMigrations
     # @see https://edgeapi.rubyonrails.org/classes/ActiveRecord/ConnectionAdapters/SchemaStatements.html#method-i-remove_check_constraint
     #
     def remove_check_constraint(table_name, expression = nil, **options)
-      if __check_constraint_exists?(table_name, expression: expression, **options)
+      if check_constraint_exists?(table_name, expression: expression, **options)
         super
       else
         Utils.say(<<~MSG.squish)
@@ -874,16 +854,14 @@ module OnlineMigrations
       end
     end
 
-    if Utils.ar_version >= 7.1
-      def add_exclusion_constraint(table_name, expression, **options)
-        if __exclusion_constraint_exists?(table_name, expression: expression, **options)
-          Utils.say(<<~MSG.squish)
-            Exclusion constraint was not created because it already exists (this may be due to an aborted migration or similar).
-            table_name: #{table_name}, expression: #{expression}
-          MSG
-        else
-          super
-        end
+    def add_exclusion_constraint(table_name, expression, **options)
+      if __exclusion_constraint_exists?(table_name, expression: expression, **options)
+        Utils.say(<<~MSG.squish)
+          Exclusion constraint was not created because it already exists (this may be due to an aborted migration or similar).
+          table_name: #{table_name}, expression: #{expression}
+        MSG
+      else
+        super
       end
     end
 
@@ -932,20 +910,9 @@ module OnlineMigrations
         end
       end
 
-      # Will not be needed for Active Record >= 7.1
-      def __index_defined_for?(index, columns = nil, name: nil, unique: nil, valid: nil, include: nil, nulls_not_distinct: nil, **options)
-        columns = options[:column] if columns.blank?
-        (columns.nil? || Array(index.columns) == Array(columns).map(&:to_s)) &&
-          (name.nil? || index.name == name.to_s) &&
-          (unique.nil? || index.unique == unique) &&
-          (valid.nil? || index.valid == valid) &&
-          (include.nil? || Array(index.include) == Array(include).map(&:to_s)) &&
-          (nulls_not_distinct.nil? || index.nulls_not_distinct == nulls_not_distinct)
-      end
-
       def __not_null_constraint_exists?(table_name, column_name, name: nil)
         name ||= __not_null_constraint_name(table_name, column_name)
-        __check_constraint_exists?(table_name, name: name)
+        check_constraint_exists?(table_name, name: name)
       end
 
       def __not_null_constraint_name(table_name, column_name)
@@ -958,21 +925,7 @@ module OnlineMigrations
 
       def __text_limit_constraint_exists?(table_name, column_name, name: nil)
         name ||= __text_limit_constraint_name(table_name, column_name)
-        __check_constraint_exists?(table_name, name: name)
-      end
-
-      # Can use index validity attribute for Active Record >= 7.1.
-      def __index_valid?(index_name, schema:)
-        select_value(<<~SQL)
-          SELECT indisvalid
-          FROM pg_index i
-          JOIN pg_class c
-            ON i.indexrelid = c.oid
-          JOIN pg_namespace n
-            ON c.relnamespace = n.oid
-          WHERE n.nspname = #{schema}
-            AND c.relname = #{quote(index_name)}
-        SQL
+        check_constraint_exists?(table_name, name: name)
       end
 
       def __copy_foreign_key(fk, to_column, **options)
@@ -996,15 +949,6 @@ module OnlineMigrations
         end
       end
 
-      # Can be replaced by native method in Active Record >= 7.1.
-      def __check_constraint_exists?(table_name, **options)
-        if !options.key?(:name) && !options.key?(:expression)
-          raise ArgumentError, "At least one of :name or :expression must be supplied"
-        end
-
-        check_constraint_for(table_name, **options).present?
-      end
-
       def __exclusion_constraint_exists?(table_name, **options)
         if !options.key?(:name) && !options.key?(:expression)
           raise ArgumentError, "At least one of :name or :expression must be supplied"

data/lib/online_migrations/shard_aware.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module OnlineMigrations
+  module ShardAware
+    extend ActiveSupport::Concern
+
+    included do
+      before_validation :set_connection_class_name
+    end
+
+    def on_shard_if_present(&block)
+      if shard
+        connection_class.connected_to(shard: shard.to_sym, role: :writing, &block)
+      else
+        yield
+      end
+    end
+
+    def connection_class_name=(value)
+      if value && (klass = value.safe_constantize)
+        if !(klass <= ActiveRecord::Base)
+          raise ArgumentError, "connection_class_name is not an ActiveRecord::Base child class"
+        end
+
+        connection_class = Utils.find_connection_class(klass)
+        super(connection_class.name)
+      end
+    end
+
+    # @private
+    def connection_class
+      if connection_class_name && (klass = connection_class_name.safe_constantize)
+        Utils.find_connection_class(klass)
+      else
+        ActiveRecord::Base
+      end
+    end
+
+    private
+      def set_connection_class_name
+        self.connection_class_name ||= "ActiveRecord::Base"
+      end
+  end
+end

data/lib/online_migrations/utils.rb

@@ -69,23 +69,6 @@ module OnlineMigrations
         end
       end
 
-      # Implementation is from ActiveRecord.
-      # This is not needed for ActiveRecord >= 7.1 (https://github.com/rails/rails/pull/47753).
-      def index_name(table_name, column_name)
-        max_index_name_size = 62
-        name = "index_#{table_name}_on_#{Array(column_name) * '_and_'}"
-        return name if name.bytesize <= max_index_name_size
-
-        # Fallback to short version, add hash to ensure uniqueness
-        hashed_identifier = "_#{OpenSSL::Digest::SHA256.hexdigest(name).first(10)}"
-        name = "idx_on_#{Array(column_name) * '_'}"
-
-        short_limit = max_index_name_size - hashed_identifier.bytesize
-        short_name = name[0, short_limit]
-
-        "#{short_name}#{hashed_identifier}"
-      end
-
       # Returns estimated rows count for a table.
       # https://www.citusdata.com/blog/2016/10/12/count-performance/
       def estimated_count(connection, table_name)
@@ -144,9 +127,7 @@ module OnlineMigrations
           # This is the way that currently is used in ActiveRecord tests themselves.
           pool_manager = ActiveRecord::Base.connection_handler.send(:get_pool_manager, ancestor.name)
 
-          # .uniq call is not needed for Active Record 7.1+
-          # See https://github.com/rails/rails/pull/49284.
-          return pool_manager.shard_names.uniq if pool_manager
+          return pool_manager.shard_names if pool_manager
         end
       end
 

data/lib/online_migrations/verbose_sql_logs.rb

@@ -14,13 +14,7 @@ module OnlineMigrations
         stdout_logger.level = @activerecord_logger_was.level
         stdout_logger = ActiveSupport::TaggedLogging.new(stdout_logger)
 
-        combined_logger =
-          # Broadcasting logs API was changed in https://github.com/rails/rails/pull/48615.
-          if Utils.ar_version >= 7.1
-            ActiveSupport::BroadcastLogger.new(stdout_logger, @activerecord_logger_was)
-          else
-            stdout_logger.extend(ActiveSupport::Logger.broadcast(@activerecord_logger_was))
-          end
+        combined_logger = ActiveSupport::BroadcastLogger.new(stdout_logger, @activerecord_logger_was)
 
         ActiveRecord::Base.logger = combined_logger
         ActiveRecord.verbose_query_logs = false

data/lib/online_migrations/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module OnlineMigrations
-  VERSION = "0.25.0"
+  VERSION = "0.27.0"
 end

data/lib/online_migrations.rb

@@ -6,7 +6,8 @@ require "online_migrations/version"
 require "online_migrations/utils"
 require "online_migrations/background_schema_migrations/migration_helpers"
 require "online_migrations/change_column_type_helpers"
-require "online_migrations/background_migrations/migration_helpers"
+require "online_migrations/background_data_migrations/migration_helpers"
+require "online_migrations/active_record_batch_enumerator"
 require "online_migrations/schema_statements"
 require "online_migrations/schema_cache"
 require "online_migrations/migration"
@@ -29,7 +30,8 @@ module OnlineMigrations
   autoload :ForeignKeysCollector
   autoload :IndexDefinition
   autoload :CommandChecker
-  autoload :BackgroundMigration
+  autoload :DataMigration
+  autoload :ShardAware
 
   autoload_at "online_migrations/lock_retrier" do
     autoload :LockRetrier
@@ -40,24 +42,21 @@ module OnlineMigrations
 
   autoload :CopyTrigger
 
-  module BackgroundMigrations
+  module BackgroundDataMigrations
     extend ActiveSupport::Autoload
 
     autoload :Config
     autoload :MigrationStatusValidator
-    autoload :MigrationJobStatusValidator
-    autoload :BackgroundMigrationClassValidator
     autoload :BackfillColumn
     autoload :CopyColumn
     autoload :DeleteAssociatedRecords
     autoload :DeleteOrphanedRecords
     autoload :PerformActionOnRelation
     autoload :ResetCounters
-    autoload :MigrationJob
     autoload :Migration
-    autoload :MigrationJobRunner
-    autoload :MigrationRunner
+    autoload :MigrationJob
     autoload :Scheduler
+    autoload :Ticker
   end
 
   module BackgroundSchemaMigrations
@@ -70,6 +69,10 @@ module OnlineMigrations
     autoload :Scheduler
   end
 
+  # Make aliases for less typing.
+  DataMigrations = BackgroundDataMigrations
+  SchemaMigrations = BackgroundSchemaMigrations
+
   class << self
     # @private
     attr_accessor :current_migration
@@ -87,10 +90,10 @@ module OnlineMigrations
     # @option options [String, Symbol, nil] :shard The name of the shard to run
     #   background data migrations on. By default runs on all shards.
     #
-    def run_background_migrations(**options)
-      BackgroundMigrations::Scheduler.run(**options)
+    def run_background_data_migrations(**options)
+      BackgroundDataMigrations::Scheduler.run(**options)
     end
-    alias run_background_data_migrations run_background_migrations
+    alias run_background_migrations run_background_data_migrations
 
     # Run background schema migrations
     #
@@ -102,12 +105,7 @@ module OnlineMigrations
     end
 
     def deprecator
-      @deprecator ||=
-        if Utils.ar_version >= 7.1
-          ActiveSupport::Deprecation.new(nil, "online_migrations")
-        else
-          ActiveSupport::Deprecation
-        end
+      @deprecator ||= ActiveSupport::Deprecation.new(nil, "online_migrations")
     end
 
     # @private
@@ -124,11 +122,13 @@ module OnlineMigrations
 
       if OnlineMigrations::Utils.ar_version >= 7.2
         ActiveRecord::ConnectionAdapters::SchemaCache.prepend(OnlineMigrations::SchemaCache72)
-      elsif OnlineMigrations::Utils.ar_version >= 7.1
-        ActiveRecord::ConnectionAdapters::SchemaCache.prepend(OnlineMigrations::SchemaCache71)
       else
         ActiveRecord::ConnectionAdapters::SchemaCache.prepend(OnlineMigrations::SchemaCache)
       end
+
+      if !ActiveRecord::Batches::BatchEnumerator.method_defined?(:use_ranges)
+        ActiveRecord::Batches::BatchEnumerator.include(OnlineMigrations::ActiveRecordBatchEnumerator)
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: online_migrations
 version: !ruby/object:Gem::Version
-  version: 0.25.0
+  version: 0.27.0
 platform: ruby
 authors:
 - fatkodima
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-02-03 00:00:00.000000000 Z
+date: 2025-05-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '7.0'
+        version: '7.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '7.0'
+        version: '7.1'
 description:
 email:
 - fatkodima123@gmail.com
@@ -34,39 +34,38 @@ files:
 - CHANGELOG.md
 - LICENSE.txt
 - README.md
+- docs/0.27-upgrade.md
 - docs/background_data_migrations.md
 - docs/background_schema_migrations.md
 - docs/configuring.md
-- lib/generators/online_migrations/background_migration_generator.rb
+- lib/generators/online_migrations/data_migration_generator.rb
 - lib/generators/online_migrations/install_generator.rb
 - lib/generators/online_migrations/templates/add_sharding_to_online_migrations.rb.tt
 - lib/generators/online_migrations/templates/add_timestamps_to_background_migrations.rb.tt
-- lib/generators/online_migrations/templates/background_data_migration.rb.tt
 - lib/generators/online_migrations/templates/background_schema_migrations_change_unique_index.rb.tt
+- lib/generators/online_migrations/templates/change_background_data_migrations.rb.tt
 - lib/generators/online_migrations/templates/create_background_schema_migrations.rb.tt
+- lib/generators/online_migrations/templates/data_migration.rb.tt
 - lib/generators/online_migrations/templates/initializer.rb.tt
 - lib/generators/online_migrations/templates/install_migration.rb.tt
 - lib/generators/online_migrations/templates/migration.rb.tt
 - lib/generators/online_migrations/upgrade_generator.rb
 - lib/online_migrations.rb
+- lib/online_migrations/active_record_batch_enumerator.rb
 - lib/online_migrations/application_record.rb
-- lib/online_migrations/background_migration.rb
-- lib/online_migrations/background_migrations/backfill_column.rb
-- lib/online_migrations/background_migrations/background_migration_class_validator.rb
-- lib/online_migrations/background_migrations/config.rb
-- lib/online_migrations/background_migrations/copy_column.rb
-- lib/online_migrations/background_migrations/delete_associated_records.rb
-- lib/online_migrations/background_migrations/delete_orphaned_records.rb
-- lib/online_migrations/background_migrations/migration.rb
-- lib/online_migrations/background_migrations/migration_helpers.rb
-- lib/online_migrations/background_migrations/migration_job.rb
-- lib/online_migrations/background_migrations/migration_job_runner.rb
-- lib/online_migrations/background_migrations/migration_job_status_validator.rb
-- lib/online_migrations/background_migrations/migration_runner.rb
-- lib/online_migrations/background_migrations/migration_status_validator.rb
-- lib/online_migrations/background_migrations/perform_action_on_relation.rb
-- lib/online_migrations/background_migrations/reset_counters.rb
-- lib/online_migrations/background_migrations/scheduler.rb
+- lib/online_migrations/background_data_migrations/backfill_column.rb
+- lib/online_migrations/background_data_migrations/config.rb
+- lib/online_migrations/background_data_migrations/copy_column.rb
+- lib/online_migrations/background_data_migrations/delete_associated_records.rb
+- lib/online_migrations/background_data_migrations/delete_orphaned_records.rb
+- lib/online_migrations/background_data_migrations/migration.rb
+- lib/online_migrations/background_data_migrations/migration_helpers.rb
+- lib/online_migrations/background_data_migrations/migration_job.rb
+- lib/online_migrations/background_data_migrations/migration_status_validator.rb
+- lib/online_migrations/background_data_migrations/perform_action_on_relation.rb
+- lib/online_migrations/background_data_migrations/reset_counters.rb
+- lib/online_migrations/background_data_migrations/scheduler.rb
+- lib/online_migrations/background_data_migrations/ticker.rb
 - lib/online_migrations/background_schema_migrations/config.rb
 - lib/online_migrations/background_schema_migrations/migration.rb
 - lib/online_migrations/background_schema_migrations/migration_helpers.rb
@@ -79,6 +78,7 @@ files:
 - lib/online_migrations/command_recorder.rb
 - lib/online_migrations/config.rb
 - lib/online_migrations/copy_trigger.rb
+- lib/online_migrations/data_migration.rb
 - lib/online_migrations/database_tasks.rb
 - lib/online_migrations/error_messages.rb
 - lib/online_migrations/foreign_keys_collector.rb
@@ -89,6 +89,7 @@ files:
 - lib/online_migrations/schema_cache.rb
 - lib/online_migrations/schema_dumper.rb
 - lib/online_migrations/schema_statements.rb
+- lib/online_migrations/shard_aware.rb
 - lib/online_migrations/utils.rb
 - lib/online_migrations/verbose_sql_logs.rb
 - lib/online_migrations/version.rb
@@ -107,7 +108,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.0'
+      version: '3.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="

data/lib/online_migrations/background_migration.rb

@@ -1,64 +0,0 @@
-# frozen_string_literal: true
-
-module OnlineMigrations
-  # Base class that is inherited by the host application's background migration classes.
-  class BackgroundMigration
-    class NotFoundError < NameError; end
-
-    class << self
-      # Finds a Background Migration with the given name.
-      #
-      # @param name [String] the name of the Background Migration to be found.
-      #
-      # @return [BackgroundMigration] the Background Migration with the given name.
-      #
-      # @raise [NotFoundError] if a Background Migration with the given name does not exist.
-      #
-      def named(name)
-        namespace = OnlineMigrations.config.background_migrations.migrations_module.constantize
-        internal_namespace = ::OnlineMigrations::BackgroundMigrations
-
-        migration = "#{namespace}::#{name}".safe_constantize ||
-                    "#{internal_namespace}::#{name}".safe_constantize
-
-        raise NotFoundError.new("Background Migration #{name} not found", name) if migration.nil?
-        if !(migration.is_a?(Class) && migration < self)
-          raise NotFoundError.new("#{name} is not a Background Migration", name)
-        end
-
-        migration
-      end
-    end
-
-    # The relation to be iterated over.
-    #
-    # @return [ActiveRecord::Relation]
-    #
-    # @raise [NotImplementedError] with a message advising subclasses to
-    #     implement an override for this method.
-    #
-    def relation
-      raise NotImplementedError, "#{self.class.name} must implement a 'relation' method"
-    end
-
-    # Processes one batch.
-    #
-    # @param _relation [ActiveRecord::Relation] the current batch from the enumerator being iterated
-    # @return [void]
-    #
-    # @raise [NotImplementedError] with a message advising subclasses to
-    #     implement an override for this method.
-    #
-    def process_batch(_relation)
-      raise NotImplementedError, "#{self.class.name} must implement a 'process_batch' method"
-    end
-
-    # Returns the count of rows that will be iterated over (optional, to be able to show progress).
-    #
-    # @return [Integer, nil, :no_count]
-    #
-    def count
-      :no_count
-    end
-  end
-end

data/lib/online_migrations/background_migrations/backfill_column.rb

@@ -1,54 +0,0 @@
-# frozen_string_literal: true
-
-module OnlineMigrations
-  module BackgroundMigrations
-    # @private
-    class BackfillColumn < BackgroundMigration
-      attr_reader :table_name, :updates, :model_name
-
-      def initialize(table_name, updates, model_name = nil)
-        @table_name = table_name
-        @updates = updates
-        @model_name = model_name
-      end
-
-      def relation
-        column, value = updates.first
-
-        if updates.size == 1 && !value.nil?
-          # If value is nil, the generated SQL is correct (`WHERE column IS NOT NULL`).
-          # Otherwise, the SQL is `WHERE column != value`. This condition ignores column
-          # with NULLs in it, so we need to also manually check for NULLs.
-          quoted_column = connection.quote_column_name(column)
-          model.unscoped.where("#{quoted_column} != ? OR #{quoted_column} IS NULL", value)
-        else
-          model.unscoped.where.not(updates)
-        end
-      end
-
-      def process_batch(relation)
-        relation.update_all(updates)
-      end
-
-      def count
-        # Exact counts are expensive on large tables, since PostgreSQL
-        # needs to do a full scan. An estimated count should give a pretty decent
-        # approximation of rows count in this case.
-        Utils.estimated_count(connection, table_name)
-      end
-
-      private
-        def model
-          @model ||= if model_name.present?
-                       Object.const_get(model_name, false)
-                     else
-                       Utils.define_model(table_name)
-                     end
-        end
-
-        def connection
-          model.connection
-        end
-    end
-  end
-end

data/lib/online_migrations/background_migrations/background_migration_class_validator.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module OnlineMigrations
-  module BackgroundMigrations
-    # @private
-    class BackgroundMigrationClassValidator < ActiveModel::Validator
-      def validate(record)
-        relation = record.migration_relation
-        migration_name = record.migration_name
-
-        if !relation.is_a?(ActiveRecord::Relation)
-          record.errors.add(
-            :migration_name,
-            "#{migration_name}#relation must return an ActiveRecord::Relation object"
-          )
-          return
-        end
-
-        if relation.arel.orders.present? || relation.arel.taken.present?
-          record.errors.add(
-            :migration_name,
-            "#{migration_name}#relation cannot use ORDER BY or LIMIT due to the way how iteration with a cursor is designed. " \
-            "You can use other ways to limit the number of rows, e.g. a WHERE condition on the primary key column."
-          )
-        end
-      end
-    end
-  end
-end