online_migrations 0.1.0

data/lib/online_migrations/background_migrations/migration_job_status_validator.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module OnlineMigrations
+  module BackgroundMigrations
+    # @private
+    class MigrationJobStatusValidator < ActiveModel::Validator
+      VALID_STATUS_TRANSITIONS = {
+        "enqueued" => ["running"],
+        "running" => ["succeeded", "failed"],
+        "failed" => ["enqueued", "running"],
+      }
+
+      def validate(record)
+        return unless (previous_status, new_status = record.status_change)
+
+        valid_new_statuses = VALID_STATUS_TRANSITIONS.fetch(previous_status, [])
+
+        unless valid_new_statuses.include?(new_status)
+          record.errors.add(
+            :status,
+            "cannot transition background migration job from status #{previous_status} to #{new_status}"
+          )
+        end
+      end
+    end
+  end
+end

data/lib/online_migrations/background_migrations/migration_runner.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module OnlineMigrations
+  module BackgroundMigrations
+    # Runs single background migration.
+    class MigrationRunner
+      attr_reader :migration
+
+      def initialize(migration)
+        @migration = migration
+      end
+
+      # Runs one background migration job.
+      def run_migration_job
+        migration.running! if migration.enqueued?
+        migration_payload = { background_migration: migration }
+
+        if !migration.migration_jobs.exists?
+          ActiveSupport::Notifications.instrument("started.background_migrations", migration_payload)
+        end
+
+        if should_throttle?
+          ActiveSupport::Notifications.instrument("throttled.background_migrations", migration_payload)
+          return
+        end
+
+        next_migration_job = find_or_create_next_migration_job
+
+        if next_migration_job
+          job_runner = MigrationJobRunner.new(next_migration_job)
+          job_runner.run
+        elsif !migration.migration_jobs.active.exists?
+          if migration.migration_jobs.failed.exists?
+            migration.failed!
+          else
+            migration.succeeded!
+          end
+
+          ActiveSupport::Notifications.instrument("completed.background_migrations", migration_payload)
+        end
+
+        next_migration_job
+      end
+
+      # Runs the background migration until completion.
+      #
+      # @note This method should not be used in production environments
+      #
+      def run_all_migration_jobs
+        raise "This method is not intended for use in production environments" if !Utils.developer_env?
+        return if migration.completed?
+
+        migration.running!
+
+        while migration.running?
+          run_migration_job
+        end
+      end
+
+      # Finishes the background migration.
+      #
+      # Keep running until the migration is finished.
+      #
+      def finish
+        return if migration.completed?
+
+        # Mark is as finishing to avoid being picked up
+        # by the background migrations scheduler.
+        migration.finishing!
+
+        while migration.finishing?
+          run_migration_job
+        end
+      end
+
+      private
+        def should_throttle?
+          ::OnlineMigrations.config.background_migrations.throttler.call
+        end
+
+        def find_or_create_next_migration_job
+          if (min_value, max_value = migration.next_batch_range)
+            create_migration_job!(min_value, max_value)
+          else
+            migration.migration_jobs.retriable.first
+          end
+        end
+
+        def create_migration_job!(min_value, max_value)
+          migration.migration_jobs.create!(
+            min_value: min_value,
+            max_value: max_value
+          )
+        end
+    end
+  end
+end

data/lib/online_migrations/background_migrations/migration_status_validator.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module OnlineMigrations
+  module BackgroundMigrations
+    # @private
+    class MigrationStatusValidator < ActiveModel::Validator
+      VALID_STATUS_TRANSITIONS = {
+        # enqueued -> running occurs when the migration starts performing.
+        # enqueued -> paused occurs when the migration is paused before starting.
+        "enqueued" => ["running", "paused"],
+        # running -> paused occurs when a user pauses the migration as
+        #   it's performing.
+        # running -> finishing occurs when a user manually finishes the migration.
+        # running -> succeeded occurs when the migration completes successfully.
+        # running -> failed occurs when the migration raises an exception when running.
+        "running" => [
+          "paused",
+          "finishing",
+          "succeeded",
+          "failed",
+        ],
+        # finishing -> succeeded occurs when the migration completes successfully.
+        # finishing -> failed occurs when the migration raises an exception when running.
+        "finishing" => ["succeeded", "failed"],
+        # paused -> running occurs when the migration is resumed after being paused.
+        "paused" => ["running"],
+        # failed -> enqueued occurs when the failed migration jobs are retried after being failed.
+        "failed" => ["enqueued"],
+      }
+
+      def validate(record)
+        return unless (previous_status, new_status = record.status_change)
+
+        valid_new_statuses = VALID_STATUS_TRANSITIONS.fetch(previous_status, [])
+
+        unless valid_new_statuses.include?(new_status)
+          record.errors.add(
+            :status,
+            "cannot transition background migration from status #{previous_status} to #{new_status}"
+          )
+        end
+      end
+    end
+  end
+end

data/lib/online_migrations/background_migrations/scheduler.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module OnlineMigrations
+  module BackgroundMigrations
+    # Class responsible for scheduling background migrations.
+    # It selects runnable background migrations and runs them one step (one batch) at a time.
+    # A migration is considered runnable if it is not completed and time the interval between
+    #   successive runs has passed.
+    # Scheduler ensures (via advisory locks) that at most one background migration at a time is running per database.
+    #
+    # Scheduler should be run via some kind of periodical means, for example, cron.
+    # @example Run via whenever
+    #   # add this to schedule.rb
+    #   every 1.minute do
+    #     runner "OnlineMigrations::BackgroundMigrations::Scheduler.run"
+    #   end
+    #
+    class Scheduler
+      def self.run
+        new.run
+      end
+
+      # Runs Scheduler
+      def run
+        active_migrations = Migration.active.queue_order
+        runnable_migrations = active_migrations.select(&:interval_elapsed?)
+
+        runnable_migrations.each do |migration|
+          connection = migration.migration_relation.connection
+
+          with_exclusive_lock(connection) do
+            run_migration_job(migration)
+          end
+        end
+      end
+
+      private
+        def with_exclusive_lock(connection, &block)
+          lock = AdvisoryLock.new(name: "online_migrations_scheduler", connection: connection)
+          lock.with_lock(&block)
+        end
+
+        def run_migration_job(migration)
+          runner = MigrationRunner.new(migration)
+          runner.run_migration_job
+        end
+    end
+  end
+end

data/lib/online_migrations/batch_iterator.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module OnlineMigrations
+  # @private
+  class BatchIterator
+    attr_reader :relation
+
+    def initialize(relation)
+      if !relation.is_a?(ActiveRecord::Relation)
+        raise ArgumentError, "relation is not an ActiveRecord::Relation"
+      end
+
+      @relation = relation
+    end
+
+    def each_batch(of: 1000, column: relation.primary_key, start: nil, finish: nil, order: :asc)
+      unless [:asc, :desc].include?(order)
+        raise ArgumentError, ":order must be :asc or :desc, got #{order.inspect}"
+      end
+
+      relation = apply_limits(self.relation, column, start, finish, order)
+
+      base_relation = relation.except(:select)
+        .select(column)
+        .reorder(column => order)
+
+      start_row = base_relation.uncached { base_relation.first }
+
+      return unless start_row
+
+      start_id = start_row[column]
+      arel_table = relation.arel_table
+
+      0.step do |index|
+        if order == :asc
+          start_cond = arel_table[column].gteq(start_id)
+        else
+          start_cond = arel_table[column].lteq(start_id)
+        end
+
+        stop_row = base_relation.uncached do
+          base_relation
+            .where(start_cond)
+            .offset(of)
+            .first
+        end
+
+        batch_relation = relation.where(start_cond)
+
+        if stop_row
+          stop_id = stop_row[column]
+          start_id = stop_id
+
+          if order == :asc
+            stop_cond = arel_table[column].lt(stop_id)
+          else
+            stop_cond = arel_table[column].gt(stop_id)
+          end
+
+          batch_relation = batch_relation.where(stop_cond)
+        end
+
+        # Any ORDER BYs are useless for this relation and can lead to less
+        # efficient UPDATE queries, hence we get rid of it.
+        batch_relation = batch_relation.except(:order)
+
+        # Retaining the results in the query cache would undermine the point of batching.
+        batch_relation.uncached { yield batch_relation, index }
+
+        break unless stop_row
+      end
+    end
+
+    private
+      def apply_limits(relation, column, start, finish, order)
+        if start
+          relation = relation.where(relation.arel_table[column].public_send((order == :asc ? :gteq : :lteq), start))
+        end
+
+        if finish
+          relation = relation.where(relation.arel_table[column].public_send((order == :asc ? :lteq : :gteq), finish))
+        end
+
+        relation
+      end
+  end
+end