online_migrations 0.26.0 → 0.27.1

data/lib/online_migrations/background_schema_migrations/migration.rb

@@ -8,13 +8,15 @@ module OnlineMigrations
     #   `enqueue_background_schema_migration` helper inside migrations.
     #
     class Migration < ApplicationRecord
+      include ShardAware
+
       STATUSES = [
-        :enqueued,    # The migration has been enqueued by the user.
-        :running,     # The migration is being performed by a migration executor.
-        :errored,     # The migration raised an error during last run.
-        :failed,      # The migration raises an error when running and retry attempts exceeded.
-        :succeeded,   # The migration finished without error.
-        :cancelled,   # The migration was cancelled by the user.
+        "enqueued",    # The migration has been enqueued by the user.
+        "running",     # The migration is being performed by a migration executor.
+        "errored",     # The migration raised an error during last run.
+        "failed",      # The migration raises an error when running and retry attempts exceeded.
+        "succeeded",   # The migration finished without error.
+        "cancelled",   # The migration was cancelled by the user.
       ]
 
       MAX_IDENTIFIER_LENGTH = 63
@@ -22,36 +24,12 @@ module OnlineMigrations
       self.table_name = :background_schema_migrations
 
       scope :queue_order, -> { order(created_at: :asc) }
-      scope :parents, -> { where(parent_id: nil) }
-      scope :runnable, -> { where(composite: false) }
       scope :active, -> { where(status: [:enqueued, :running, :errored]) }
-      scope :except_succeeded, -> { where.not(status: :succeeded) }
-
-      scope :stuck, -> do
-        runnable.active.where(<<~SQL)
-          updated_at <= NOW() - interval '1 second' * (COALESCE(statement_timeout, 60*60*24) + 60*10)
-        SQL
-      end
-
-      scope :retriable, -> do
-        stuck_sql = connection.unprepared_statement { stuck.to_sql }
-
-        from(Arel.sql(<<~SQL))
-          (
-            (SELECT * FROM background_schema_migrations WHERE NOT composite AND status = 'errored')
-            UNION
-            (#{stuck_sql})
-          ) AS #{table_name}
-        SQL
-      end
 
       alias_attribute :name, :migration_name
 
       enum :status, STATUSES.index_with(&:to_s)
 
-      belongs_to :parent, class_name: name, optional: true, inverse_of: :children
-      has_many :children, class_name: name, foreign_key: :parent_id, inverse_of: :parent
-
       validates :table_name, presence: true, length: { maximum: MAX_IDENTIFIER_LENGTH }
       validates :definition, presence: true
       validates :migration_name, presence: true, uniqueness: {
@@ -65,70 +43,53 @@ module OnlineMigrations
         end,
       }
 
-      validate :validate_children_statuses, if: -> { composite? && status_changed? }
-      validate :validate_connection_class, if: :connection_class_name?
       validate :validate_table_exists
       validates_with MigrationStatusValidator, on: :update
 
       before_validation :set_defaults
 
+      # Returns whether the migration is completed, which is defined as
+      # having a status of succeeded, failed, or cancelled.
+      #
+      # @return [Boolean] whether the migration is completed.
+      #
       def completed?
-        succeeded? || failed?
+        succeeded? || failed? || cancelled?
       end
 
-      # Overwrite enum's generated method to correctly work for composite migrations.
-      def cancelled!
-        return super if !composite?
-
-        transaction do
-          super
-          children.each { |child| child.cancelled! if !child.succeeded? }
-        end
+      # Returns whether the migration is active, which is defined as
+      # having a status of enqueued, or running.
+      #
+      # @return [Boolean] whether the migration is active.
+      #
+      def active?
+        enqueued? || running?
       end
+
       alias cancel cancelled!
 
+      # Returns whether this migration is pausable.
+      #
       def pausable?
         false
       end
 
-      def can_be_paused?
-        false
-      end
-
-      def can_be_cancelled?
-        !succeeded? && !cancelled?
-      end
-
-      # Returns the progress of the background schema migration.
+      # Dummy method to support the same interface as background data migrations.
       #
-      # @return [Float] value in range from 0.0 to 100.0
+      # @return [nil]
       #
       def progress
-        if succeeded?
-          100.0
-        elsif composite?
-          progresses = children.map(&:progress)
-          # There should not be composite migrations without children,
-          # but children may be deleted for some reason, so we need to
-          # make a check to avoid 0 division error.
-          if progresses.any?
-            (progresses.sum.to_f / progresses.size).round(2)
-          else
-            0.0
-          end
-        else
-          0.0
-        end
       end
 
       # Whether the migration is considered stuck (is running for some configured time).
       #
       def stuck?
-        # Composite migrations are not considered stuck.
-        return false if composite?
-
-        stuck_timeout = (statement_timeout || 1.day) + 10.minutes
-        running? && updated_at <= stuck_timeout.seconds.ago
+        if index_addition?
+          running? && !index_build_in_progress?
+        else
+          stuck_timeout = (statement_timeout || 1.day) + 10.minutes
+          running? && updated_at <= stuck_timeout.seconds.ago
+        end
       end
 
       # Mark this migration as ready to be processed again.
@@ -136,27 +97,16 @@ module OnlineMigrations
       # This is used to manually retrying failed migrations.
       #
       def retry
-        if composite? && failed?
-          transaction do
-            update!(status: :enqueued, finished_at: nil)
-            children.failed.each(&:retry)
-          end
-
-          true
-        elsif failed?
-          transaction do
-            parent.update!(status: :enqueued, finished_at: nil) if parent
-
-            update!(
-              status: :enqueued,
-              attempts: 0,
-              started_at: nil,
-              finished_at: nil,
-              error_class: nil,
-              error_message: nil,
-              backtrace: nil
-            )
-          end
+        if failed?
+          update!(
+            status: :enqueued,
+            attempts: 0,
+            started_at: nil,
+            finished_at: nil,
+            error_class: nil,
+            error_message: nil,
+            backtrace: nil
+          )
 
           true
         else
@@ -168,15 +118,6 @@ module OnlineMigrations
         definition.match?(/create (unique )?index/i)
       end
 
-      # @private
-      def connection_class
-        if connection_class_name && (klass = connection_class_name.safe_constantize)
-          Utils.find_connection_class(klass)
-        else
-          ActiveRecord::Base
-        end
-      end
-
       # @private
       def attempts_exceeded?
         attempts >= max_attempts
@@ -184,7 +125,7 @@ module OnlineMigrations
 
       # @private
       def run
-        on_shard do
+        on_shard_if_present do
           connection = connection_class.connection
 
           connection.with_lock_retries do
@@ -214,28 +155,11 @@ module OnlineMigrations
       end
 
       private
-        def validate_children_statuses
-          if composite?
-            if succeeded? && children.except_succeeded.exists?
-              errors.add(:base, "all child migrations must be succeeded")
-            elsif failed? && !children.failed.exists?
-              errors.add(:base, "at least one child migration must be failed")
-            end
-          end
-        end
-
-        def validate_connection_class
-          klass = connection_class_name.safe_constantize
-          if !(klass <= ActiveRecord::Base)
-            errors.add(:connection_class_name, "is not an ActiveRecord::Base child class")
-          end
-        end
-
         def validate_table_exists
           # Skip this validation if we have invalid connection class name.
           return if errors.include?(:connection_class_name)
 
-          on_shard do
+          on_shard_if_present do
             if !connection_class.connection.table_exists?(table_name)
               errors.add(:table_name, "'#{table_name}' does not exist")
             end
@@ -248,9 +172,13 @@ module OnlineMigrations
           self.statement_timeout ||= config.statement_timeout
         end
 
-        def on_shard(&block)
-          shard = (self.shard || connection_class.default_shard).to_sym
-          connection_class.connected_to(shard: shard, role: :writing, &block)
+        def index_build_in_progress?
+          indexes_in_progress = connection_class.connection.select_values(<<~SQL)
+            SELECT index_relid::regclass::text
+            FROM pg_stat_progress_create_index
+          SQL
+
+          indexes_in_progress.include?(name)
         end
 
         def with_statement_timeout(connection, timeout)

data/lib/online_migrations/background_schema_migrations/migration_helpers.rb

@@ -78,67 +78,46 @@ module OnlineMigrations
       #   ensure_background_schema_migration_succeeded("index_users_on_email")
       #
       def ensure_background_schema_migration_succeeded(migration_name)
-        migration = Migration.parents.find_by(migration_name: migration_name)
+        migrations = Migration.where(migration_name: migration_name).to_a
 
-        if migration.nil?
-          Utils.raise_in_prod_or_say_in_dev("Could not find background schema migration: '#{migration_name}'")
-        elsif !migration.succeeded?
-          raise "Expected background schema migration '#{migration_name}' to be marked as 'succeeded', " \
-                "but it is '#{migration.status}'."
+        if migrations.empty?
+          Utils.raise_in_prod_or_say_in_dev("Could not find background schema migration(s): '#{migration_name}'.")
+        elsif !migrations.all?(&:succeeded?)
+          raise "Expected background schema migration(s) '#{migration_name}' to be marked as 'succeeded'."
         end
       end
 
-      def enqueue_background_schema_migration(name, table_name, **options)
-        if options[:connection_class_name].nil? && Utils.multiple_databases?
-          raise ArgumentError, "You must pass a :connection_class_name when using multiple databases."
-        end
-
-        migration = create_background_schema_migration(name, table_name, **options)
+      def enqueue_background_schema_migration(migration_name, table_name, connection_class_name: nil, **options)
+        options.assert_valid_keys(:definition, :max_attempts, :statement_timeout)
 
-        run_inline = OnlineMigrations.config.run_background_migrations_inline
-        if run_inline && run_inline.call
-          runner = MigrationRunner.new(migration)
-          runner.run
+        if Utils.multiple_databases? && !connection_class_name
+          raise ArgumentError, "You must pass a :connection_class_name when using multiple databases."
         end
 
-        migration
-      end
-
-      # @private
-      def create_background_schema_migration(migration_name, table_name, connection_class_name: nil, **options)
-        options.assert_valid_keys(:definition, :max_attempts, :statement_timeout)
-
         if connection_class_name
-          connection_class_name = __normalize_connection_class_name(connection_class_name)
+          klass = connection_class_name.constantize
+          connection_class = Utils.find_connection_class(klass)
+          # Normalize to the real connection class name.
+          connection_class_name = connection_class.name
+        else
+          connection_class = ActiveRecord::Base
         end
 
-        Migration.find_or_create_by!(migration_name: migration_name, shard: nil,
-                                     connection_class_name: connection_class_name) do |migration|
-          migration.assign_attributes(**options, table_name: table_name)
+        shards = Utils.shard_names(connection_class)
+        shards = [nil] if shards.size == 1
 
-          shards = Utils.shard_names(migration.connection_class)
-          if shards.size > 1
-            migration.children = shards.map do |shard|
-              child = migration.dup
-              child.shard = shard
-              child
-            end
+        shards.each do |shard|
+          migration = Migration.create_with(**options, table_name: table_name)
+                               .find_or_create_by!(migration_name: migration_name, shard: shard, connection_class_name: connection_class_name)
 
-            migration.composite = true
+          if Utils.run_background_migrations_inline?
+            runner = MigrationRunner.new(migration)
+            runner.run
           end
         end
-      end
 
-      private
-        def __normalize_connection_class_name(connection_class_name)
-          if connection_class_name
-            klass = connection_class_name.safe_constantize
-            if klass
-              connection_class = Utils.find_connection_class(klass)
-              connection_class.name if connection_class
-            end
-          end
-        end
+        true
+      end
     end
   end
 end

data/lib/online_migrations/background_schema_migrations/migration_runner.rb

@@ -13,119 +13,65 @@ module OnlineMigrations
       def run
         return if migration.cancelled? || migration.succeeded?
 
-        mark_as_running if migration.enqueued? || migration.errored?
+        migration.running! if migration.enqueued? || migration.errored?
+        migration_payload = { migration: migration }
 
-        if migration.composite?
-          migration.children.each do |child_migration|
-            runner = self.class.new(child_migration)
-            runner.run
-          end
+        if migration.attempts == 0
+          ActiveSupport::Notifications.instrument("started.background_schema_migrations", migration_payload)
         else
-          do_run
+          ActiveSupport::Notifications.instrument("retried.background_schema_migrations", migration_payload)
         end
-      end
-
-      private
-        def mark_as_running
-          Migration.transaction do
-            migration.running!
 
-            if (parent = migration.parent)
-              if parent.started_at
-                parent.update!(status: :running, finished_at: nil)
-              else
-                parent.update!(status: :running, started_at: Time.current, finished_at: nil)
-              end
-            end
-          end
+        if should_throttle?
+          ActiveSupport::Notifications.instrument("throttled.background_schema_migrations", migration_payload)
+          return
         end
 
-        def do_run
-          migration_payload = notifications_payload(migration)
-
-          if migration.attempts == 0
-            ActiveSupport::Notifications.instrument("started.background_schema_migrations", migration_payload)
-          else
-            ActiveSupport::Notifications.instrument("retried.background_schema_migrations", migration_payload)
-          end
-
-          if should_throttle?
-            ActiveSupport::Notifications.instrument("throttled.background_schema_migrations", migration_payload)
-            return
-          end
-
-          migration.update!(
-            attempts: migration.attempts + 1,
-            status: :running,
-            started_at: Time.current,
-            finished_at: nil,
-            error_class: nil,
-            error_message: nil,
-            backtrace: nil
-          )
-
-          ActiveSupport::Notifications.instrument("run.background_schema_migrations", migration_payload) do
-            migration.run
-          end
-
-          # Background schema migrations could take a while to run. It is possible, that the process
-          # never reaches this (or the rescue below) line of code. E.g., when it is force quitted
-          # (SIGKILL etc.) and so the migration will end up in the "running" state and the query is
-          # still executing (or already finished) in the database. This migration can either be safely
-          # manually retried or will be picked up in the future by scheduler when it decides that
-          # this migration is stuck.
-
-          migration.update!(status: :succeeded, finished_at: Time.current)
+        migration.update!(
+          attempts: migration.attempts + 1,
+          status: :running,
+          started_at: Time.current,
+          finished_at: nil,
+          error_class: nil,
+          error_message: nil,
+          backtrace: nil
+        )
+
+        ActiveSupport::Notifications.instrument("run.background_schema_migrations", migration_payload) do
+          migration.run
+        end
 
-          ActiveSupport::Notifications.instrument("completed.background_schema_migrations", migration_payload)
+        # Background schema migrations could take a while to run. It is possible, that the process
+        # never reaches this (or the rescue below) line of code. E.g., when it is force quitted
+        # (SIGKILL etc.) and so the migration will end up in the "running" state and the query is
+        # still executing (or already finished) in the database. This migration can either be safely
+        # manually retried or will be picked up in the future by scheduler when it decides that
+        # this migration is stuck.
 
-          complete_parent_if_needed(migration) if migration.parent.present?
-        rescue Exception => e # rubocop:disable Lint/RescueException
-          backtrace_cleaner = ::OnlineMigrations.config.backtrace_cleaner
+        migration.update!(status: :succeeded, finished_at: Time.current)
 
-          status = migration.attempts_exceeded? ? :failed : :errored
+        ActiveSupport::Notifications.instrument("completed.background_schema_migrations", migration_payload)
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        backtrace_cleaner = ::OnlineMigrations.config.backtrace_cleaner
 
-          migration.update!(
-            status: status,
-            finished_at: Time.current,
-            error_class: e.class.name,
-            error_message: e.message,
-            backtrace: backtrace_cleaner ? backtrace_cleaner.clean(e.backtrace) : e.backtrace
-          )
+        status = migration.attempts_exceeded? ? :failed : :errored
 
-          complete_parent_if_needed(migration) if migration.parent.present?
+        migration.update!(
+          status: status,
+          finished_at: Time.current,
+          error_class: e.class.name,
+          error_message: e.message,
+          backtrace: backtrace_cleaner ? backtrace_cleaner.clean(e.backtrace) : e.backtrace
+        )
 
-          ::OnlineMigrations.config.background_schema_migrations.error_handler.call(e, migration)
-          raise if Utils.run_background_migrations_inline?
-        end
+        ::OnlineMigrations.config.background_schema_migrations.error_handler.call(e, migration)
+        raise if Utils.run_background_migrations_inline?
+      end
 
+      private
         def should_throttle?
           ::OnlineMigrations.config.throttler.call
         end
-
-        def complete_parent_if_needed(migration)
-          parent = migration.parent
-          completed = false
-
-          parent.with_lock do
-            children = parent.children.select(:status)
-            if children.all?(&:succeeded?)
-              parent.update!(status: :succeeded, finished_at: Time.current)
-              completed = true
-            elsif children.any?(&:failed?)
-              parent.update!(status: :failed, finished_at: Time.current)
-              completed = true
-            end
-          end
-
-          if completed
-            ActiveSupport::Notifications.instrument("completed.background_migrations", notifications_payload(migration))
-          end
-        end
-
-        def notifications_payload(migration)
-          { background_schema_migration: migration }
-        end
     end
   end
 end

data/lib/online_migrations/background_schema_migrations/scheduler.rb

@@ -35,8 +35,8 @@ module OnlineMigrations
 
       private
         def find_migration(**options)
-          active_migrations = Migration.running.reject(&:stuck?)
-          runnable_migrations = Migration.runnable.enqueued.queue_order.to_a + Migration.retriable.queue_order.to_a
+          stuck_migrations, active_migrations = Migration.running.partition(&:stuck?)
+          runnable_migrations = (Migration.enqueued + Migration.errored + stuck_migrations).sort_by(&:created_at)
 
           if options.key?(:shard)
             runnable_migrations = runnable_migrations.select { |migration| migration.shard.to_s == options[:shard].to_s }

data/lib/online_migrations/change_column_type_helpers.rb

@@ -152,7 +152,11 @@ module OnlineMigrations
     #   revert_initialize_column_type_change(:files, :size)
     #
     def revert_initialize_column_type_change(table_name, column_name, _new_type = nil, **_options)
-      cleanup_column_type_change(table_name, column_name)
+      tmp_column_name = __change_type_column(column_name)
+      transaction do
+        __remove_copy_triggers(table_name, column_name, tmp_column_name)
+        remove_column(table_name, tmp_column_name)
+      end
     end
 
     # Same as `revert_initialize_column_type_change` but for multiple columns.
@@ -160,7 +164,12 @@ module OnlineMigrations
     #
     def revert_initialize_columns_type_change(table_name, columns_and_types, **_options)
       column_names = columns_and_types.map(&:first)
-      cleanup_columns_type_change(table_name, *column_names)
+      tmp_column_names = column_names.map { |column_name| __change_type_column(column_name) }
+
+      transaction do
+        __remove_copy_triggers(table_name, column_names, tmp_column_names)
+        remove_columns(table_name, *tmp_column_names)
+      end
     end
 
     # Backfills data from the old column to the new column.
@@ -347,6 +356,7 @@ module OnlineMigrations
     #   the original column type to be able to revert.
     #
     def cleanup_column_type_change(table_name, column_name)
+      __ensure_not_in_transaction!
       cleanup_columns_type_change(table_name, column_name)
     end
 
@@ -354,6 +364,8 @@ module OnlineMigrations
     # @see #cleanup_column_type_change
     #
     def cleanup_columns_type_change(table_name, *column_names)
+      __ensure_not_in_transaction!
+
       tmp_column_names = column_names.map { |column_name| __change_type_column(column_name) }
 
       # Safely remove existing indexes and foreign keys first, if any.
@@ -477,10 +489,11 @@ module OnlineMigrations
       end
 
       def __copy_check_constraints(table_name, from_column, to_column)
-        check_constraints = check_constraints(table_name).select { |c| c.expression.include?(from_column) }
+        from_column_re = /\b#{from_column}\b/
+        check_constraints = check_constraints(table_name).select { |c| c.expression.match?(from_column_re) }
 
         check_constraints.each do |check|
-          new_expression = check.expression.gsub(from_column, to_column)
+          new_expression = check.expression.gsub(from_column_re, to_column)
 
           add_check_constraint(table_name, new_expression, validate: false)
 

data/lib/online_migrations/config.rb

@@ -210,10 +210,10 @@ module OnlineMigrations
 
     # Configuration object to configure background migrations
     #
-    # @return [BackgroundMigrationsConfig]
-    # @see BackgroundMigrationsConfig
+    # @return [BackgroundDataMigrations::Config]
+    # @see BackgroundDataMigrations::Config
     #
-    attr_reader :background_migrations
+    attr_reader :background_data_migrations
 
     attr_reader :background_schema_migrations
 
@@ -230,7 +230,7 @@ module OnlineMigrations
         lock_timeout: 0.2.seconds
       )
 
-      @background_migrations = BackgroundMigrations::Config.new
+      @background_data_migrations = BackgroundDataMigrations::Config.new
       @background_schema_migrations = BackgroundSchemaMigrations::Config.new
 
       @checks = []