online_migrations 0.14.1 → 0.16.0

data/lib/generators/online_migrations/templates/migration.rb.tt

@@ -1,51 +1,10 @@
-class InstallOnlineMigrations < <%= migration_parent %>
-  def change
-    create_table :background_migrations do |t|
-      t.bigint :parent_id
-      t.string :migration_name, null: false
-      t.jsonb :arguments, default: [], null: false
-      t.string :batch_column_name, null: false
-      t.bigint :min_value, null: false
-      t.bigint :max_value, null: false
-      t.bigint :rows_count
-      t.integer :batch_size, null: false
-      t.integer :sub_batch_size, null: false
-      t.integer :batch_pause, null: false
-      t.integer :sub_batch_pause_ms, null: false
-      t.integer :batch_max_attempts, null: false
-      t.string :status, default: "enqueued", null: false
-      t.string :shard
-      t.boolean :composite, default: false, null: false
-      t.timestamps
-
-      t.foreign_key :background_migrations, column: :parent_id, on_delete: :cascade
-
-      t.index [:migration_name, :arguments, :shard],
-        unique: true, name: :index_background_migrations_on_unique_configuration
-    end
-
-    create_table :background_migration_jobs do |t|
-      t.bigint :migration_id, null: false
-      t.bigint :min_value, null: false
-      t.bigint :max_value, null: false
-      t.integer :batch_size, null: false
-      t.integer :sub_batch_size, null: false
-      t.integer :pause_ms, null: false
-      t.datetime :started_at
-      t.datetime :finished_at
-      t.string :status, default: "enqueued", null: false
-      t.integer :max_attempts, null: false
-      t.integer :attempts, default: 0, null: false
-      t.string :error_class
-      t.string :error_message
-      t.string :backtrace, array: true
-      t.timestamps
-
-      t.foreign_key :background_migrations, column: :migration_id, on_delete: :cascade
+class Enqueue<%= class_name %> < <%= migration_parent %>
+  def up
+    enqueue_background_migration("<%= class_name %>", ...args)
+  end
 
-      t.index [:migration_id, :max_value], name: :index_background_migration_jobs_on_max_value
-      t.index [:migration_id, :status, :updated_at], name: :index_background_migration_jobs_on_updated_at
-      t.index [:migration_id, :finished_at], name: :index_background_migration_jobs_on_finished_at
-    end
+  def down
+    # Make sure to pass the same arguments as in the "up" method, if any.
+    remove_background_migration("<%= class_name %>", ...args)
   end
 end

data/lib/generators/online_migrations/upgrade_generator.rb

@@ -23,6 +23,11 @@ module OnlineMigrations
 
         migrations = []
         migrations << "add_sharding_to_online_migrations" if !columns.include?("shard")
+
+        if !connection.table_exists?(BackgroundSchemaMigrations::Migration.table_name)
+          migrations << "create_background_schema_migrations"
+        end
+
         migrations
       end
 

data/lib/online_migrations/application_record.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module OnlineMigrations
+  # Base class for all records used by this gem.
+  #
+  # Can be extended to setup different database where all tables related to
+  # online_migrations will live.
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end

data/lib/online_migrations/background_migrations/background_migration_class_validator.rb

@@ -16,13 +16,6 @@ module OnlineMigrations
           return
         end
 
-        if relation.joins_values.present? && !record.batch_column_name.to_s.include?(".")
-          record.errors.add(
-            :batch_column_name,
-            "must be a fully-qualified column if you join a table"
-          )
-        end
-
         if relation.arel.orders.present? || relation.arel.taken.present?
           record.errors.add(
             :migration_name,

data/lib/online_migrations/background_migrations/config.rb

@@ -39,17 +39,13 @@ module OnlineMigrations
       #
       attr_accessor :batch_max_attempts
 
-      # Allows to throttle background migrations based on external signal (e.g. database health)
-      #
-      # It will be called before each batch run.
-      # If throttled, the current run will be retried next time.
-      #
-      # @return [Proc]
-      #
-      # @example
-      #   OnlineMigrations.config.background_migrations.throttler = -> { DatabaseStatus.unhealthy? }
-      #
-      attr_reader :throttler
+      def throttler
+        OnlineMigrations.deprecator.warn(<<~MSG)
+          `config.background_migrations.throttler` is deprecated and will be removed.
+          Use `config.throttler` instead.
+        MSG
+        OnlineMigrations.config.throttler
+      end
 
       # The number of seconds that must pass before the running job is considered stuck
       #
@@ -57,13 +53,21 @@ module OnlineMigrations
       #
       attr_accessor :stuck_jobs_timeout
 
-      # The Active Support backtrace cleaner that will be used to clean the
-      # backtrace of a migration job that errors.
-      #
-      # @return [ActiveSupport::BacktraceCleaner, nil] the backtrace cleaner to
-      #   use when cleaning a job's backtrace. Defaults to `Rails.backtrace_cleaner`
-      #
-      attr_accessor :backtrace_cleaner
+      def backtrace_cleaner
+        OnlineMigrations.deprecator.warn(<<~MSG)
+          `config.background_migrations.backtrace_cleaner` is deprecated and will be removed.
+          Use `config.backtrace_cleaner` instead.
+        MSG
+        OnlineMigrations.config.backtrace_cleaner
+      end
+
+      def backtrace_cleaner=(value)
+        OnlineMigrations.deprecator.warn(<<~MSG)
+          `config.background_migrations.backtrace_cleaner=` is deprecated and will be removed.
+          Use `config.backtrace_cleaner=` instead.
+        MSG
+        OnlineMigrations.config.backtrace_cleaner = value
+      end
 
       # The callback to perform when an error occurs in the migration job.
       #
@@ -86,17 +90,16 @@ module OnlineMigrations
         @batch_pause = 0.seconds
         @sub_batch_pause_ms = 100
         @batch_max_attempts = 5
-        @throttler = -> { false }
         @stuck_jobs_timeout = 1.hour
         @error_handler = ->(error, errored_job) {}
       end
 
       def throttler=(value)
-        if !value.respond_to?(:call)
-          raise ArgumentError, "background_migrations throttler must be a callable."
-        end
-
-        @throttler = value
+        OnlineMigrations.deprecator.warn(<<~MSG)
+          `config.background_migrations.throttler=` is deprecated and will be removed.
+          Use `config.throttler=` instead.
+        MSG
+        OnlineMigrations.config.throttler = value
       end
     end
   end

data/lib/online_migrations/background_migrations/delete_associated_records.rb

@@ -21,7 +21,7 @@ module OnlineMigrations
       end
 
       def process_batch(relation)
-        relation.delete_all(:delete_all)
+        relation.delete_all
       end
     end
   end

data/lib/online_migrations/background_migrations/migration.rb

@@ -181,7 +181,7 @@ module OnlineMigrations
 
       # @private
       def on_shard(&block)
-        abstract_class = find_abstract_class(migration_model)
+        abstract_class = Utils.find_connection_class(migration_model)
 
         shard = (self.shard || abstract_class.default_shard).to_sym
         abstract_class.connected_to(shard: shard, role: :writing, &block)
@@ -290,13 +290,6 @@ module OnlineMigrations
             min_value
           end
         end
-
-        def find_abstract_class(model)
-          model.ancestors.find do |parent|
-            parent == ActiveRecord::Base ||
-              (parent.is_a?(Class) && parent.abstract_class?)
-          end
-        end
     end
   end
 end

data/lib/online_migrations/background_migrations/migration_helpers.rb

@@ -372,8 +372,7 @@ module OnlineMigrations
       def enqueue_background_migration(migration_name, *arguments, **options)
         migration = create_background_migration(migration_name, *arguments, **options)
 
-        run_inline = OnlineMigrations.config.run_background_migrations_inline
-        if run_inline && run_inline.call
+        if Utils.run_background_migrations_inline?
           runner = MigrationRunner.new(migration)
           runner.run_all_migration_jobs
         end
@@ -381,11 +380,26 @@ module OnlineMigrations
         migration
       end
 
+      # Removes the background migration for the given class name and arguments, if exists.
+      #
+      # @param migration_name [String, Class] Background migration job class name
+      # @param arguments [Array] Extra arguments the migration was originally created with
+      #
+      # @example
+      #   remove_background_migration("BackfillProjectIssuesCount")
+      #
+      def remove_background_migration(migration_name, *arguments)
+        migration_name = migration_name.name if migration_name.is_a?(Class)
+        Migration.for_configuration(migration_name, arguments).delete_all
+      end
+
       # @private
       def create_background_migration(migration_name, *arguments, **options)
         options.assert_valid_keys(:batch_column_name, :min_value, :max_value, :batch_size, :sub_batch_size,
             :batch_pause, :sub_batch_pause_ms, :batch_max_attempts)
 
+        migration_name = migration_name.name if migration_name.is_a?(Class)
+
         migration = Migration.new(
           migration_name: migration_name,
           arguments: arguments,

data/lib/online_migrations/background_migrations/migration_job.rb

@@ -44,7 +44,7 @@ module OnlineMigrations
         enum status: STATUSES.index_with(&:to_s)
       end
 
-      delegate :migration_class, :migration_object, :migration_relation, :batch_column_name,
+      delegate :migration_name, :migration_class, :migration_object, :migration_relation, :batch_column_name,
         :arguments, :batch_pause, to: :migration
 
       belongs_to :migration
@@ -73,7 +73,10 @@ module OnlineMigrations
           status: self.class.statuses[:enqueued],
           attempts: 0,
           started_at: nil,
-          finished_at: nil
+          finished_at: nil,
+          error_class: nil,
+          error_message: nil,
+          backtrace: nil
         )
       end
 

data/lib/online_migrations/background_migrations/migration_job_runner.rb

@@ -35,7 +35,7 @@ module OnlineMigrations
 
         migration_job.update!(status: :succeeded, finished_at: Time.current)
       rescue Exception => e # rubocop:disable Lint/RescueException
-        backtrace_cleaner = ::OnlineMigrations.config.background_migrations.backtrace_cleaner
+        backtrace_cleaner = ::OnlineMigrations.config.backtrace_cleaner
 
         migration_job.update!(
           status: :failed,
@@ -46,6 +46,7 @@ module OnlineMigrations
         )
 
         ::OnlineMigrations.config.background_migrations.error_handler.call(e, migration_job)
+        raise if Utils.run_background_migrations_inline?
       end
 
       private

data/lib/online_migrations/background_migrations/migration_runner.rb

@@ -105,7 +105,7 @@ module OnlineMigrations
         end
 
         def should_throttle?
-          ::OnlineMigrations.config.background_migrations.throttler.call
+          ::OnlineMigrations.config.throttler.call
         end
 
         def find_or_create_next_migration_job

data/lib/online_migrations/background_schema_migrations/config.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module OnlineMigrations
+  module BackgroundSchemaMigrations
+    # Class representing configuration options for background schema migrations.
+    class Config
+      # Maximum number of run attempts
+      #
+      # When attempts are exhausted, the migration is marked as failed.
+      # @return [Integer] defaults to 5
+      #
+      attr_accessor :max_attempts
+
+      # Statement timeout value used when running background schema migration.
+      #
+      # @return [Integer] defaults to 1 hour
+      #
+      attr_accessor :statement_timeout
+
+      # The callback to perform when an error occurs in the migration.
+      #
+      # @example
+      #   OnlineMigrations.config.background_schema_migrations.error_handler = ->(error, errored_migration) do
+      #     Bugsnag.notify(error) do |notification|
+      #       notification.add_metadata(:background_schema_migration, { name: errored_migration.name })
+      #     end
+      #   end
+      #
+      # @return [Proc] the callback to perform when an error occurs in the migration
+      #
+      attr_accessor :error_handler
+
+      def initialize
+        @max_attempts = 5
+        @statement_timeout = 1.hour
+        @error_handler = ->(error, errored_migration) {}
+      end
+    end
+  end
+end

data/lib/online_migrations/background_schema_migrations/migration.rb

@@ -0,0 +1,205 @@
+# frozen_string_literal: true
+
+module OnlineMigrations
+  module BackgroundSchemaMigrations
+    # Class representing background schema migration.
+    #
+    # @note The records of this class should not be created manually, but via
+    #   `enqueue_background_schema_migration` helper inside migrations.
+    #
+    class Migration < ApplicationRecord
+      STATUSES = [
+        :enqueued,    # The migration has been enqueued by the user.
+        :running,     # The migration is being performed by a migration executor.
+        :failed,      # The migration raises an exception when running.
+        :succeeded,   # The migration finished without error.
+      ]
+
+      MAX_IDENTIFIER_LENGTH = 63
+
+      self.table_name = :background_schema_migrations
+
+      scope :queue_order, -> { order(created_at: :asc) }
+      scope :runnable, -> { where(composite: false) }
+      scope :active, -> { where(status: [statuses[:enqueued], statuses[:running]]) }
+      scope :except_succeeded, -> { where.not(status: :succeeded) }
+
+      scope :stuck, -> do
+        runnable.active.where(<<~SQL)
+          updated_at <= NOW() - interval '1 second' * (COALESCE(statement_timeout, 60*60*24) + 60*10)
+        SQL
+      end
+
+      scope :retriable, -> do
+        failed_retriable = runnable.failed.where("attempts < max_attempts")
+
+        stuck_sql             = connection.unprepared_statement { stuck.to_sql }
+        failed_retriable_sql  = connection.unprepared_statement { failed_retriable.to_sql }
+
+        from(Arel.sql(<<~SQL))
+          (
+            (#{failed_retriable_sql})
+            UNION
+            (#{stuck_sql})
+          ) AS #{table_name}
+        SQL
+      end
+
+      alias_attribute :name, :migration_name
+
+      # Avoid deprecation warnings.
+      if Utils.ar_version >= 7
+        enum :status, STATUSES.index_with(&:to_s)
+      else
+        enum status: STATUSES.index_with(&:to_s)
+      end
+
+      belongs_to :parent, class_name: name, optional: true
+      has_many :children, class_name: name, foreign_key: :parent_id
+
+      validates :migration_name, presence: true, uniqueness: { scope: :shard }
+      validates :table_name, presence: true, length: { maximum: MAX_IDENTIFIER_LENGTH }
+      validates :definition, presence: true
+
+      validate :validate_children_statuses, if: -> { composite? && status_changed? }
+      validate :validate_connection_class, if: :connection_class_name?
+      validate :validate_table_exists
+      validates_with MigrationStatusValidator, on: :update
+
+      before_validation :set_defaults
+
+      def completed?
+        succeeded? || failed?
+      end
+
+      # Returns the progress of the background schema migration.
+      #
+      # @return [Float] value in range from 0.0 to 100.0
+      #
+      def progress
+        if succeeded?
+          100.0
+        elsif composite?
+          progresses = children.map(&:progress)
+          (progresses.sum.to_f / progresses.size).round(2)
+        else
+          0.0
+        end
+      end
+
+      # Mark this migration as ready to be processed again.
+      #
+      # This is used to manually retrying failed migrations.
+      #
+      def retry
+        if composite?
+          children.failed.each(&:retry)
+        elsif failed?
+          update!(
+            status: self.class.statuses[:enqueued],
+            attempts: 0,
+            started_at: nil,
+            finished_at: nil,
+            error_class: nil,
+            error_message: nil,
+            backtrace: nil
+          )
+        end
+      end
+
+      # @private
+      def connection_class
+        if connection_class_name && (klass = connection_class_name.safe_constantize)
+          Utils.find_connection_class(klass)
+        else
+          ActiveRecord::Base
+        end
+      end
+
+      # @private
+      def attempts_exceeded?
+        attempts >= max_attempts
+      end
+
+      # @private
+      def run
+        on_shard do
+          connection = connection_class.connection
+
+          connection.with_lock_retries do
+            statement_timeout = self.statement_timeout || OnlineMigrations.config.statement_timeout
+
+            with_statement_timeout(connection, statement_timeout) do
+              case definition
+              when /create (unique )?index/i
+                index = connection.indexes(table_name).find { |i| i.name == name }
+                if index
+                  # Use index validity from https://github.com/rails/rails/pull/45160
+                  # when switching to ActiveRecord >= 7.1.
+                  schema = connection.send(:__schema_for_table, table_name)
+                  if connection.send(:__index_valid?, name, schema: schema)
+                    return
+                  else
+                    connection.remove_index(table_name, name: name)
+                  end
+                end
+              end
+
+              connection.execute(definition)
+            end
+          end
+        end
+      end
+
+      private
+        def validate_children_statuses
+          if composite?
+            if succeeded? && children.except_succeeded.exists?
+              errors.add(:base, "all child migrations must be succeeded")
+            elsif failed? && !children.failed.exists?
+              errors.add(:base, "at least one child migration must be failed")
+            end
+          end
+        end
+
+        def validate_connection_class
+          klass = connection_class_name.safe_constantize
+          if !(klass < ActiveRecord::Base)
+            errors.add(:connection_class_name, "is not an ActiveRecord::Base child class")
+          end
+        end
+
+        def validate_table_exists
+          # Skip this validation if we have invalid connection class name.
+          return if errors.include?(:connection_class_name)
+
+          on_shard do
+            if !connection_class.connection.table_exists?(table_name)
+              errors.add(:table_name, "'#{table_name}' does not exist")
+            end
+          end
+        end
+
+        def set_defaults
+          config = ::OnlineMigrations.config.background_schema_migrations
+          self.max_attempts ||= config.max_attempts
+          self.statement_timeout ||= config.statement_timeout
+        end
+
+        def on_shard(&block)
+          shard = (self.shard || connection_class.default_shard).to_sym
+          connection_class.connected_to(shard: shard, role: :writing, &block)
+        end
+
+        def with_statement_timeout(connection, timeout)
+          return yield if timeout.nil?
+
+          prev_value = connection.select_value("SHOW statement_timeout")
+          connection.execute("SET statement_timeout TO #{connection.quote(timeout.in_milliseconds)}")
+          yield
+        ensure
+          connection.execute("SET statement_timeout TO #{connection.quote(prev_value)}")
+        end
+    end
+  end
+end

data/lib/online_migrations/background_schema_migrations/migration_helpers.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module OnlineMigrations
+  module BackgroundSchemaMigrations
+    module MigrationHelpers
+      def add_index_in_background(table_name, column_name, **options)
+        migration_options = options.extract!(:max_attempts, :statement_timeout, :connection_class_name)
+
+        if index_exists?(table_name, column_name, **options)
+          Utils.say("Index creation was not enqueued because the index already exists.")
+          return
+        end
+
+        options[:algorithm] = :concurrently
+        index, algorithm, if_not_exists = add_index_options(table_name, column_name, **options)
+
+        create_index = ActiveRecord::ConnectionAdapters::CreateIndexDefinition.new(index, algorithm, if_not_exists)
+        schema_creation = ActiveRecord::ConnectionAdapters::PostgreSQL::SchemaCreation.new(self)
+        definition = schema_creation.accept(create_index)
+
+        enqueue_background_schema_migration(index.name, table_name, definition: definition, **migration_options)
+      end
+
+      def remove_index_in_background(table_name, column_name = nil, name:, **options)
+        raise ArgumentError, "Index name must be specified" if name.blank?
+
+        migration_options = options.extract!(:max_attempts, :statement_timeout, :connection_class_name)
+
+        if !index_exists?(table_name, column_name, **options, name: name)
+          Utils.say("Index deletion was not enqueued because the index does not exist.")
+          return
+        end
+
+        definition = "DROP INDEX CONCURRENTLY IF EXISTS #{quote_column_name(name)}"
+        enqueue_background_schema_migration(name, table_name, definition: definition, **migration_options)
+      end
+
+      def enqueue_background_schema_migration(name, table_name, **options)
+        if options[:connection_class_name].nil? && Utils.multiple_databases?
+          raise ArgumentError, "You must pass a :connection_class_name when using multiple databases."
+        end
+
+        migration = create_background_schema_migration(name, table_name, **options)
+
+        run_inline = OnlineMigrations.config.run_background_migrations_inline
+        if run_inline && run_inline.call
+          runner = MigrationRunner.new(migration)
+          runner.run
+        end
+
+        migration
+      end
+
+      # @private
+      def create_background_schema_migration(migration_name, table_name, **options)
+        options.assert_valid_keys(:definition, :max_attempts, :statement_timeout, :connection_class_name)
+        migration = Migration.new(migration_name: migration_name, table_name: table_name, **options)
+
+        shards = Utils.shard_names(migration.connection_class)
+        if shards.size > 1
+          migration.children = shards.map do |shard|
+            child = migration.dup
+            child.shard = shard
+            child
+          end
+
+          migration.composite = true
+        end
+
+        # This will save all the records using a transaction.
+        migration.save!
+        migration
+      end
+    end
+  end
+end