online_migrations 0.26.0 → 0.27.0

data/lib/online_migrations/background_migrations/migration_job.rb

@@ -1,109 +0,0 @@
-# frozen_string_literal: true
-
-module OnlineMigrations
-  module BackgroundMigrations
-    class MigrationJob < ApplicationRecord
-      STATUSES = [
-        :enqueued,
-        :running,
-        :errored,
-        :failed,
-        :succeeded,
-        :cancelled,
-      ]
-
-      self.table_name = :background_migration_jobs
-
-      scope :active, -> { where(status: [:enqueued, :running, :errored]) }
-      scope :completed, -> { where(status: [:failed, :succeeded]) }
-      scope :stuck, -> do
-        timeout = OnlineMigrations.config.background_migrations.stuck_jobs_timeout
-        running.where("updated_at <= ?", timeout.seconds.ago)
-      end
-
-      scope :retriable, -> do
-        stuck_sql = connection.unprepared_statement { stuck.to_sql }
-
-        from(Arel.sql(<<~SQL))
-          (
-            (SELECT * FROM background_migration_jobs WHERE status = 'errored')
-            UNION
-            (#{stuck_sql})
-          ) AS #{table_name}
-        SQL
-      end
-
-      scope :except_succeeded, -> { where.not(status: :succeeded) }
-
-      enum :status, STATUSES.index_with(&:to_s)
-
-      delegate :migration_name, :migration_class, :migration_object, :migration_relation, :batch_column_name,
-        :arguments, :batch_pause, to: :migration
-
-      belongs_to :migration, inverse_of: :migration_jobs
-
-      validates :min_value, :max_value, presence: true, numericality: { greater_than: 0 }
-      validate :values_in_migration_range, if: :min_value?
-      validate :validate_values_order, if: :min_value?
-
-      validates_with MigrationJobStatusValidator, on: :update
-
-      before_create :copy_settings_from_migration
-
-      # Whether the job is considered stuck (is running for some configured time).
-      #
-      def stuck?
-        timeout = OnlineMigrations.config.background_migrations.stuck_jobs_timeout
-        running? && updated_at <= timeout.seconds.ago
-      end
-
-      def attempts_exceeded?
-        attempts >= max_attempts
-      end
-
-      # Mark this job as ready to be processed again.
-      #
-      # This is used when retrying failed jobs.
-      #
-      def retry
-        if failed?
-          transaction do
-            update!(
-              status: self.class.statuses[:enqueued],
-              attempts: 0,
-              started_at: nil,
-              finished_at: nil,
-              error_class: nil,
-              error_message: nil,
-              backtrace: nil
-            )
-            migration.enqueued! if migration.failed?
-          end
-          true
-        else
-          false
-        end
-      end
-
-      private
-        def values_in_migration_range
-          if min_value < migration.min_value || max_value > migration.max_value
-            errors.add(:base, "min_value and max_value should be in background migration values range")
-          end
-        end
-
-        def validate_values_order
-          if max_value.to_i < min_value.to_i
-            errors.add(:base, "max_value should be greater than or equal to min_value")
-          end
-        end
-
-        def copy_settings_from_migration
-          self.batch_size       = migration.batch_size
-          self.sub_batch_size   = migration.sub_batch_size
-          self.pause_ms         = migration.sub_batch_pause_ms
-          self.max_attempts     = migration.batch_max_attempts
-        end
-    end
-  end
-end

data/lib/online_migrations/background_migrations/migration_job_runner.rb

@@ -1,66 +0,0 @@
-# frozen_string_literal: true
-
-module OnlineMigrations
-  module BackgroundMigrations
-    # @private
-    class MigrationJobRunner
-      attr_reader :migration_job
-
-      delegate :migration, :attempts, :migration_relation, :migration_object, :sub_batch_size,
-        :batch_column_name, :min_value, :max_value, :pause_ms, to: :migration_job
-
-      def initialize(migration_job)
-        @migration_job = migration_job
-      end
-
-      def run
-        job_payload = { background_migration_job: migration_job }
-        if migration_job.attempts >= 1
-          ActiveSupport::Notifications.instrument("retried.background_migrations", job_payload)
-        end
-
-        migration_job.update!(
-          attempts: attempts + 1,
-          status: :running,
-          started_at: Time.current,
-          finished_at: nil,
-          error_class: nil,
-          error_message: nil,
-          backtrace: nil
-        )
-
-        ActiveSupport::Notifications.instrument("process_batch.background_migrations", job_payload) do
-          migration.on_shard { run_batch }
-        end
-
-        migration_job.update!(status: :succeeded, finished_at: Time.current)
-      rescue Exception => e # rubocop:disable Lint/RescueException
-        backtrace_cleaner = ::OnlineMigrations.config.backtrace_cleaner
-
-        status = migration_job.attempts_exceeded? ? :failed : :errored
-
-        migration_job.update!(
-          status: status,
-          finished_at: Time.current,
-          error_class: e.class.name,
-          error_message: e.message,
-          backtrace: backtrace_cleaner ? backtrace_cleaner.clean(e.backtrace) : e.backtrace
-        )
-
-        ::OnlineMigrations.config.background_migrations.error_handler.call(e, migration_job)
-        raise if Utils.run_background_migrations_inline?
-      end
-
-      private
-        def run_batch
-          iterator = ::OnlineMigrations::BatchIterator.new(migration_relation)
-
-          iterator.each_batch(of: sub_batch_size, column: batch_column_name,
-                              start: min_value, finish: max_value) do |sub_batch|
-            migration_object.process_batch(sub_batch)
-            sleep(pause_ms * 0.001) if pause_ms > 0
-          end
-        end
-    end
-  end
-end

data/lib/online_migrations/background_migrations/migration_job_status_validator.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module OnlineMigrations
-  module BackgroundMigrations
-    # @private
-    class MigrationJobStatusValidator < ActiveModel::Validator
-      VALID_STATUS_TRANSITIONS = {
-        "enqueued" => ["running", "cancelled"],
-        "running" => ["succeeded", "errored", "failed", "cancelled"],
-        "errored" => ["running", "failed", "cancelled"],
-        "failed" => ["enqueued", "running", "cancelled"],
-      }
-
-      def validate(record)
-        return if !record.status_changed?
-
-        previous_status, new_status = record.status_change
-        valid_new_statuses = VALID_STATUS_TRANSITIONS.fetch(previous_status, [])
-
-        if !valid_new_statuses.include?(new_status)
-          record.errors.add(
-            :status,
-            "cannot transition background migration job from status #{previous_status} to #{new_status}"
-          )
-        end
-      end
-    end
-  end
-end

data/lib/online_migrations/background_migrations/migration_runner.rb

@@ -1,161 +0,0 @@
-# frozen_string_literal: true
-
-module OnlineMigrations
-  module BackgroundMigrations
-    # Runs single background migration.
-    class MigrationRunner
-      attr_reader :migration
-
-      def initialize(migration)
-        @migration = migration
-      end
-
-      # Runs one background migration job.
-      def run_migration_job
-        raise "Should not be called on a composite (with sharding) migration" if migration.composite?
-        return if migration.cancelled? || migration.succeeded?
-
-        mark_as_running if migration.enqueued?
-        migration_payload = notifications_payload(migration)
-
-        if !migration.migration_jobs.exists?
-          ActiveSupport::Notifications.instrument("started.background_migrations", migration_payload)
-        end
-
-        if should_throttle?
-          ActiveSupport::Notifications.instrument("throttled.background_migrations", migration_payload)
-          return
-        end
-
-        next_migration_job = find_or_create_next_migration_job
-
-        if next_migration_job
-          job_runner = MigrationJobRunner.new(next_migration_job)
-          job_runner.run
-        elsif !migration.migration_jobs.active.exists?
-          if migration.migration_jobs.failed.exists?
-            migration.update!(status: :failed, finished_at: Time.current)
-          else
-            migration.update!(status: :succeeded, finished_at: Time.current)
-          end
-
-          ActiveSupport::Notifications.instrument("completed.background_migrations", migration_payload)
-
-          complete_parent_if_needed(migration) if migration.parent.present?
-        end
-
-        next_migration_job
-      end
-
-      # Runs the background migration until completion.
-      #
-      # @note This method should not be used in production environments
-      #
-      def run_all_migration_jobs
-        run_inline = OnlineMigrations.config.run_background_migrations_inline
-        if run_inline && !run_inline.call
-          raise "This method is not intended for use in production environments"
-        end
-
-        return if migration.completed? || migration.cancelled?
-
-        mark_as_running
-
-        if migration.composite?
-          migration.children.each do |child_migration|
-            runner = self.class.new(child_migration)
-            runner.run_all_migration_jobs
-          end
-        else
-          while migration.running?
-            run_migration_job
-          end
-        end
-      end
-
-      # Finishes the background migration.
-      #
-      # Keep running until the migration is finished.
-      #
-      def finish
-        return if migration.completed? || migration.cancelled?
-
-        if migration.composite?
-          migration.children.each do |child_migration|
-            runner = self.class.new(child_migration)
-            runner.finish
-          end
-        else
-          # Mark is as finishing to avoid being picked up
-          # by the background migrations scheduler.
-          migration.finishing!
-          migration.reset_failed_jobs_attempts
-
-          while migration.finishing?
-            run_migration_job
-          end
-        end
-      end
-
-      private
-        def mark_as_running
-          Migration.transaction do
-            migration.update!(status: :running, started_at: Time.current, finished_at: nil)
-
-            if (parent = migration.parent)
-              if parent.started_at
-                parent.update!(status: :running, finished_at: nil)
-              else
-                parent.update!(status: :running, started_at: Time.current, finished_at: nil)
-              end
-            end
-          end
-        end
-
-        def should_throttle?
-          ::OnlineMigrations.config.throttler.call
-        end
-
-        def find_or_create_next_migration_job
-          min_value, max_value = migration.next_batch_range
-
-          if min_value && max_value
-            create_migration_job!(min_value, max_value)
-          else
-            migration.migration_jobs.enqueued.first || migration.migration_jobs.retriable.first
-          end
-        end
-
-        def create_migration_job!(min_value, max_value)
-          migration.migration_jobs.create!(
-            min_value: min_value,
-            max_value: max_value
-          )
-        end
-
-        def complete_parent_if_needed(migration)
-          parent = migration.parent
-          completed = false
-
-          parent.with_lock do
-            children = parent.children.select(:status)
-            if children.all?(&:succeeded?)
-              parent.update!(status: :succeeded, finished_at: Time.current)
-              completed = true
-            elsif children.any?(&:failed?)
-              parent.update!(status: :failed, finished_at: Time.current)
-              completed = true
-            end
-          end
-
-          if completed
-            ActiveSupport::Notifications.instrument("completed.background_migrations", notifications_payload(migration))
-          end
-        end
-
-        def notifications_payload(migration)
-          { background_migration: migration }
-        end
-    end
-  end
-end

data/lib/online_migrations/background_migrations/migration_status_validator.rb

@@ -1,48 +0,0 @@
-# frozen_string_literal: true
-
-module OnlineMigrations
-  module BackgroundMigrations
-    # @private
-    class MigrationStatusValidator < ActiveModel::Validator
-      VALID_STATUS_TRANSITIONS = {
-        # enqueued -> running occurs when the migration starts performing.
-        # enqueued -> paused occurs when the migration is paused before starting.
-        "enqueued" => ["running", "paused", "cancelled"],
-        # running -> paused occurs when a user pauses the migration as
-        #   it's performing.
-        # running -> finishing occurs when a user manually finishes the migration.
-        # running -> succeeded occurs when the migration completes successfully.
-        # running -> failed occurs when the migration raises an exception when running.
-        "running" => [
-          "paused",
-          "finishing",
-          "succeeded",
-          "failed",
-          "cancelled",
-        ],
-        # finishing -> succeeded occurs when the migration completes successfully.
-        # finishing -> failed occurs when the migration raises an exception when running.
-        "finishing" => ["succeeded", "failed", "cancelled"],
-        # paused -> running occurs when the migration is resumed after being paused.
-        "paused" => ["running", "cancelled"],
-        # failed -> enqueued occurs when the failed migration jobs are retried after being failed.
-        # failed -> running occurs when the failed migration is retried.
-        "failed" => ["enqueued", "running", "cancelled"],
-      }
-
-      def validate(record)
-        return if !record.status_changed?
-
-        previous_status, new_status = record.status_change
-        valid_new_statuses = VALID_STATUS_TRANSITIONS.fetch(previous_status, [])
-
-        if !valid_new_statuses.include?(new_status)
-          record.errors.add(
-            :status,
-            "cannot transition background migration from status #{previous_status} to #{new_status}"
-          )
-        end
-      end
-    end
-  end
-end

data/lib/online_migrations/background_migrations/scheduler.rb

@@ -1,42 +0,0 @@
-# frozen_string_literal: true
-
-module OnlineMigrations
-  module BackgroundMigrations
-    # Class responsible for scheduling background migrations.
-    #
-    # It selects a single runnable background migration and runs it one step (one batch) at a time.
-    # A migration is considered runnable if it is not completed and the time interval between
-    # successive runs has passed.
-    #
-    # Scheduler should be configured to run periodically, for example, via cron.
-    #
-    # @example Run via whenever
-    #   # add this to schedule.rb
-    #   every 1.minute do
-    #     runner "OnlineMigrations.run_background_data_migrations"
-    #   end
-    #
-    # @example Run via whenever (specific shard)
-    #   every 1.minute do
-    #     runner "OnlineMigrations.run_background_data_migrations(shard: :shard_two)"
-    #   end
-    #
-    class Scheduler
-      def self.run(**options)
-        new.run(**options)
-      end
-
-      # Runs Scheduler
-      def run(**options)
-        active_migrations = Migration.runnable.active.queue_order
-        active_migrations = active_migrations.where(shard: options[:shard]) if options.key?(:shard)
-        runnable_migration = active_migrations.select(&:interval_elapsed?).first
-
-        if runnable_migration
-          runner = MigrationRunner.new(runnable_migration)
-          runner.run_migration_job
-        end
-      end
-    end
-  end
-end