online_migrations 0.1.0

data/Rakefile

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+require "rdoc/task"
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = "rdoc"
+  rdoc.title    = "OnlineMigrations"
+  rdoc.options << "--line-numbers"
+  rdoc.rdoc_files.include("README.md")
+  rdoc.rdoc_files.include("BACKGROUND_MIGRATIONS.md")
+  rdoc.rdoc_files.include("lib/**/*.rb")
+end
+
+task default: :test

data/gemfiles/activerecord_42.gemfile

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+@ar_gem_requirement = "~> 4.2.0"
+@pg_gem_requirement = "< 1"
+
+eval_gemfile "../Gemfile"

data/gemfiles/activerecord_50.gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+@ar_gem_requirement = "~> 5.0.0"
+
+eval_gemfile "../Gemfile"

data/gemfiles/activerecord_51.gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+@ar_gem_requirement = "~> 5.1.0"
+
+eval_gemfile "../Gemfile"

data/gemfiles/activerecord_52.gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+@ar_gem_requirement = "~> 5.2.0"
+
+eval_gemfile "../Gemfile"

data/gemfiles/activerecord_60.gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+@ar_gem_requirement = "~> 6.0.0"
+
+eval_gemfile "../Gemfile"

data/gemfiles/activerecord_61.gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+@ar_gem_requirement = "~> 6.1.0"
+
+eval_gemfile "../Gemfile"

data/gemfiles/activerecord_70.gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+@ar_gem_requirement = "~> 7.0.0"
+
+eval_gemfile "../Gemfile"

data/gemfiles/activerecord_head.gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+@ar_gem_requirement = { github: "rails/rails", branch: "main" }
+
+eval_gemfile "../Gemfile"

data/lib/generators/online_migrations/background_migration_generator.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module OnlineMigrations
+  # @private
+  class BackgroundMigrationGenerator < Rails::Generators::NamedBase
+    source_root File.expand_path("templates", __dir__)
+    desc "This generator creates a background migration file."
+
+    def create_background_migration_file
+      template_file = File.join(
+        "lib/#{migrations_module_file_path}",
+        class_path,
+        "#{file_name}.rb"
+      )
+      template("background_migration.rb", template_file)
+    end
+
+    private
+      def migrations_module_file_path
+        migrations_module.underscore
+      end
+
+      def migrations_module
+        OnlineMigrations.config.background_migrations.migrations_module
+      end
+  end
+end

data/lib/generators/online_migrations/install_generator.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/active_record/migration"
+
+module OnlineMigrations
+  # @private
+  class InstallGenerator < Rails::Generators::Base
+    include ActiveRecord::Generators::Migration
+
+    source_root File.expand_path("templates", __dir__)
+
+    def create_migration_file
+      migration_template("migration.rb", File.join(migrations_dir, "install_online_migrations.rb"))
+    end
+
+    def copy_initializer_file
+      template("initializer.rb", "config/initializers/online_migrations.rb")
+    end
+
+    private
+      def migration_parent
+        Utils.migration_parent_string
+      end
+
+      def start_after
+        self.class.next_migration_number(migrations_dir)
+      end
+
+      def migrations_dir
+        Utils.ar_version >= 5.1 ? db_migrate_path : "db/migrate"
+      end
+  end
+end

data/lib/generators/online_migrations/templates/background_migration.rb.tt

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module <%= migrations_module %>
+<% module_namespacing do -%>
+  class <%= class_name %> < OnlineMigrations::BackgroundMigration
+    def relation
+      # ActiveRecord::Relation to be iterated over
+    end
+
+    def process_batch(relation)
+      # The work to be done in a single iteration of the background migration.
+      # This should be idempotent, as the same batch may be processed more
+      # than once if the background migration is interrupted and resumed.
+    end
+
+    def count
+      # Optionally, define the number of rows that will be iterated over.
+      # This is used to track the background migration's progress.
+    end
+  end
+<% end -%>
+end

data/lib/generators/online_migrations/templates/initializer.rb.tt

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+OnlineMigrations.configure do |config|
+  # Configure the migration version starting after which checks are performed.
+  # config.start_after = <%= start_after %>
+
+  # Set the version of the production database so the right checks are run in development.
+  # config.target_version = 10
+
+  # Configure whether to perform checks when migrating down.
+  config.check_down = false
+
+  # Configure custom error messages.
+  # error_messages is a Hash with keys - error names and values - error messages.
+  # config.error_messages[:remove_column] = "Your custom instructions"
+
+  # Maximum allowed lock timeout value (in seconds).
+  # If set lock timeout is greater than this value, the migration will fail.
+  # config.lock_timeout_limit = 10.seconds
+
+  # Configure list of tables with permanently small number of records.
+  # This tables are usually tables like "settings", "prices", "plans" etc.
+  # It is considered safe to perform most of the dangerous operations on them.
+  # config.small_tables = []
+
+  # Disable specific checks.
+  # For the list of available checks look at `lib/error_messages` folder.
+  # config.disable_check(:remove_index)
+
+  # Enable specific checks. All checks are enabled by default,
+  # but this may change in the future.
+  # For the list of available checks look at `lib/error_messages` folder.
+  # config.enable_check(:remove_index)
+
+  # Lock retries.
+  # Configure your custom lock retrier (see LockRetrier).
+  # To disable lock retries, set `lock_retrier` to `nil`.
+  config.lock_retrier = OnlineMigrations::ExponentialLockRetrier.new(
+    attempts: 30,                # attempt 30 retries
+    base_delay: 0.01.seconds,    # starting with delay of 10ms between each unsuccessful try, increasing exponentially
+    max_delay: 1.minute,         # up to the maximum delay of 1 minute
+    lock_timeout: 0.05.seconds   # and 50ms set as lock timeout for each try
+  )
+
+  # Configure tables that are in the process of being renamed.
+  # config.table_renames["users"] = "clients"
+
+  # Configure columns that are in the process of being renamed.
+  # config.column_renames["users] = { "name" => "first_name" }
+
+  # Add custom checks. Use the `stop!` method to stop migrations.
+  #
+  #   config.add_check do |method, args|
+  #     if method == :add_column && args[0].to_s == "users"
+  #       stop!("No more columns on the users table")
+  #     end
+  #   end
+
+  # ==> Background migrations configuration
+  # The number of rows to process in a single background migration run.
+  # config.backround_migrations.batch_size = 20_000
+
+  # The smaller batches size that the batches will be divided into.
+  # config.backround_migrations.sub_batch_size = 1000
+
+  # The pause interval between each background migration job's execution (in seconds).
+  # config.backround_migrations.batch_pause = 0.seconds
+
+  # The number of milliseconds to sleep between each sub_batch execution.
+  # config.backround_migrations.sub_batch_pause_ms = 100
+
+  # Maximum number of batch run attempts.
+  # When attempts are exhausted, the individual batch is marked as failed.
+  # config.backround_migrations.batch_max_attempts = 5
+
+  # Configure custom throttler for background migrations.
+  # It will be called before each batch run.
+  # If throttled, the current run will be retried next time.
+  # config.backround_migrations.throttler = -> { DatabaseStatus.unhealthy? }
+
+  # The number of seconds that must pass before the running job is considered stuck.
+  # config.background_migrations.stuck_jobs_timeout = 1.hour
+
+  # The Active Support backtrace cleaner that will be used to clean the
+  # backtrace of a migration job that errors.
+  config.background_migrations.backtrace_cleaner = Rails.backtrace_cleaner
+
+  # The callback to perform when an error occurs in the migration job.
+  # config.backround_migrations.error_handler = ->(error, errored_job) do
+  #   Bugsnag.notify(error) do |notification|
+  #     notification.add_metadata(:background_migration, { name: errored_job.migration_name })
+  #   end
+  # end
+end

data/lib/generators/online_migrations/templates/migration.rb.tt

@@ -0,0 +1,46 @@
+class InstallOnlineMigrations < <%= migration_parent %>
+  def change
+    create_table :background_migrations do |t|
+      t.string :migration_name, null: false
+      t.jsonb :arguments, default: [], null: false
+      t.string :batch_column_name, null: false
+      t.bigint :min_value, null: false
+      t.bigint :max_value, null: false
+      t.bigint :rows_count
+      t.integer :batch_size, null: false
+      t.integer :sub_batch_size, null: false
+      t.integer :batch_pause, null: false
+      t.integer :sub_batch_pause_ms, null: false
+      t.integer :batch_max_attempts, null: false
+      t.string :status, default: "enqueued", null: false
+      t.timestamps null: false
+
+      t.index [:migration_name, :arguments],
+        unique: true, name: :index_background_migrations_on_unique_configuration
+    end
+
+    create_table :background_migration_jobs do |t|
+      t.bigint :migration_id, null: false
+      t.bigint :min_value, null: false
+      t.bigint :max_value, null: false
+      t.integer :batch_size, null: false
+      t.integer :sub_batch_size, null: false
+      t.integer :pause_ms, null: false
+      t.datetime :started_at
+      t.datetime :finished_at
+      t.string :status, default: "enqueued", null: false
+      t.integer :max_attempts, null: false
+      t.integer :attempts, default: 0, null: false
+      t.string :error_class
+      t.string :error_message
+      t.string :backtrace, array: true
+      t.timestamps null: false
+
+      t.foreign_key :background_migrations, column: :migration_id, on_delete: :cascade
+
+      t.index [:migration_id, :max_value], name: :index_background_migration_jobs_on_max_value
+      t.index [:migration_id, :status, :updated_at], name: :index_background_migration_jobs_on_updated_at
+      t.index [:migration_id, :finished_at], name: :index_background_migration_jobs_on_finished_at
+    end
+  end
+end

data/lib/online_migrations/background_migration.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module OnlineMigrations
+  # Base class that is inherited by the host application's background migration classes.
+  class BackgroundMigration
+    class NotFoundError < NameError; end
+
+    class << self
+      # Finds a Background Migration with the given name.
+      #
+      # @param name [String] the name of the Background Migration to be found.
+      #
+      # @return [BackgroundMigration] the Background Migration with the given name.
+      #
+      # @raise [NotFoundError] if a Background Migration with the given name does not exist.
+      #
+      def named(name)
+        namespace = OnlineMigrations.config.background_migrations.migrations_module.constantize
+        internal_namespace = ::OnlineMigrations::BackgroundMigrations
+
+        migration = "#{namespace}::#{name}".safe_constantize ||
+                    "#{internal_namespace}::#{name}".safe_constantize
+
+        raise NotFoundError.new("Background Migration #{name} not found", name) unless migration
+        unless migration.is_a?(Class) && migration < self
+          raise NotFoundError.new("#{name} is not a Background Migration", name)
+        end
+
+        migration
+      end
+    end
+
+    # The relation to be iterated over.
+    #
+    # @return [ActiveRecord::Relation]
+    #
+    # @raise [NotImplementedError] with a message advising subclasses to
+    #     implement an override for this method.
+    #
+    def relation
+      raise NotImplementedError, "#{self.class.name} must implement a 'relation' method"
+    end
+
+    # Processes one batch.
+    #
+    # @param _relation [ActiveRecord::Relation] the current batch from the enumerator being iterated
+    # @return [void]
+    #
+    # @raise [NotImplementedError] with a message advising subclasses to
+    #     implement an override for this method.
+    #
+    def process_batch(_relation)
+      raise NotImplementedError, "#{self.class.name} must implement a 'process_batch' method"
+    end
+
+    # Returns the count of rows that will be iterated over (optional, to be able to show progress).
+    #
+    # @return [Integer, nil, :no_count]
+    #
+    def count
+      :no_count
+    end
+  end
+end

data/lib/online_migrations/background_migrations/advisory_lock.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "zlib"
+
+module OnlineMigrations
+  module BackgroundMigrations
+    # @private
+    class AdvisoryLock
+      attr_reader :name, :connection
+
+      def initialize(name:, connection: ActiveRecord::Base.connection)
+        @name = name
+        @connection = connection
+      end
+
+      def try_lock
+        locked = connection.select_value("SELECT pg_try_advisory_lock(#{lock_key})")
+        Utils.to_bool(locked)
+      end
+
+      def unlock
+        connection.select_value("SELECT pg_advisory_unlock(#{lock_key})")
+      end
+
+      # Runs the given block if an advisory lock is able to be acquired.
+      def with_lock
+        if try_lock
+          begin
+            yield
+          ensure
+            unlock
+          end
+        end
+      end
+
+      def active?
+        objid = lock_key & 0xffffffff
+        classid = (lock_key & (0xffffffff << 32)) >> 32
+
+        active = connection.select_value(<<~SQL)
+          SELECT granted
+          FROM pg_locks
+          WHERE locktype = 'advisory'
+            AND pid = pg_backend_pid()
+            AND mode = 'ExclusiveLock'
+            AND classid = #{classid}
+            AND objid = #{objid}
+        SQL
+
+        Utils.to_bool(active)
+      end
+
+      private
+        SALT = 936723412
+
+        def lock_key
+          name_hash = Zlib.crc32(name)
+          SALT * name_hash
+        end
+    end
+  end
+end

data/lib/online_migrations/background_migrations/backfill_column.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module OnlineMigrations
+  module BackgroundMigrations
+    # @private
+    class BackfillColumn < BackgroundMigration
+      attr_reader :table_name, :updates, :model_name
+
+      def initialize(table_name, updates, model_name = nil)
+        @table_name = table_name
+        @updates = updates
+        @model_name = model_name
+      end
+
+      def relation
+        if updates.size == 1 && (column, value = updates.first) && !value.nil?
+          # If value is nil, the generated SQL is correct (`WHERE column IS NOT NULL`).
+          # Otherwise, the SQL is `WHERE column != value`. This condition ignores column
+          # with NULLs in it, so we need to also manually check for NULLs.
+          quoted_column = connection.quote_column_name(column)
+          model.where("#{quoted_column} != ? OR #{quoted_column} IS NULL", value)
+        else
+          Utils.ar_where_not_multiple_conditions(model, updates)
+        end
+      end
+
+      def process_batch(relation)
+        relation.update_all(updates)
+      end
+
+      def count
+        # Exact counts are expensive on large tables, since PostgreSQL
+        # needs to do a full scan. An estimated count should give a pretty decent
+        # approximation of rows count in this case.
+        Utils.estimated_count(connection, table_name)
+      end
+
+      private
+        def model
+          @model ||= if model_name.present?
+                       Object.const_get(model_name, false)
+                     else
+                       Utils.define_model(ActiveRecord::Base.connection, table_name)
+                     end
+        end
+
+        def connection
+          model.connection
+        end
+    end
+  end
+end

data/lib/online_migrations/background_migrations/background_migration_class_validator.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module OnlineMigrations
+  module BackgroundMigrations
+    # @private
+    class BackgroundMigrationClassValidator < ActiveModel::Validator
+      def validate(record)
+        relation = record.migration_relation
+        migration_name = record.migration_name
+
+        unless relation.is_a?(ActiveRecord::Relation)
+          record.errors.add(
+            :migration_name,
+            "#{migration_name}#relation must return an ActiveRecord::Relation object"
+          )
+          return
+        end
+
+        if relation.joins_values.present? && !record.batch_column_name.to_s.include?(".")
+          record.errors.add(
+            :batch_column_name,
+            "must be a fully-qualified column if you join a table"
+          )
+        end
+
+        if relation.arel.orders.present? || relation.arel.taken.present?
+          record.errors.add(
+            :migration_name,
+            "#{migration_name}#relation cannot use ORDER BY or LIMIT due to the way how iteration with a cursor is designed. " \
+              "You can use other ways to limit the number of rows, e.g. a WHERE condition on the primary key column."
+          )
+        end
+      end
+    end
+  end
+end

data/lib/online_migrations/background_migrations/config.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module OnlineMigrations
+  module BackgroundMigrations
+    # Class representing configuration options for background migrations.
+    class Config
+      # The module to namespace background migrations in
+      # @return [String] defaults to "OnlineMigrations::BackgroundMigrations"
+      attr_accessor :migrations_module
+
+      # The number of rows to process in a single background migration run
+      # @return [Integer] defaults to 20_000
+      #
+      attr_accessor :batch_size
+
+      # The smaller batches size that the batches will be divided into
+      # @return [Integer] defaults to 1000
+      #
+      attr_accessor :sub_batch_size
+
+      # The pause interval between each background migration job's execution (in seconds)
+      # @return [Integer] defaults to 0
+      #
+      attr_accessor :batch_pause
+
+      # The number of milliseconds to sleep between each sub_batch execution
+      # @return [Integer] defaults to 100 milliseconds
+      #
+      attr_accessor :sub_batch_pause_ms
+
+      # Maximum number of batch run attempts
+      #
+      # When attempts are exhausted, the individual batch is marked as failed.
+      # @return [Integer] defaults to 5
+      #
+      attr_accessor :batch_max_attempts
+
+      # Allows to throttle background migrations based on external signal (e.g. database health)
+      #
+      # It will be called before each batch run.
+      # If throttled, the current run will be retried next time.
+      #
+      # @return [Proc]
+      #
+      # @example
+      #   OnlineMigrations.config.backround_migrations.throttler = -> { DatabaseStatus.unhealthy? }
+      #
+      attr_reader :throttler
+
+      # The number of seconds that must pass before the running job is considered stuck
+      #
+      # @return [Integer] defaults to 1 hour
+      #
+      attr_accessor :stuck_jobs_timeout
+
+      # The Active Support backtrace cleaner that will be used to clean the
+      # backtrace of a migration job that errors.
+      #
+      # @return [ActiveSupport::BacktraceCleaner, nil] the backtrace cleaner to
+      #   use when cleaning a job's backtrace. Defaults to `Rails.backtrace_cleaner`
+      #
+      attr_accessor :backtrace_cleaner
+
+      # The callback to perform when an error occurs in the migration job.
+      #
+      # @example
+      #   OnlineMigrations.config.backround_migrations.error_handler = ->(error, errored_job) do
+      #     Bugsnag.notify(error) do |notification|
+      #       notification.add_metadata(:background_migration, { name: errored_job.migration_name })
+      #     end
+      #   end
+      #
+      # @return [Proc] the callback to perform when an error occurs in the migration job
+      #
+      attr_accessor :error_handler
+
+      def initialize
+        @migrations_module = "OnlineMigrations::BackgroundMigrations"
+        @batch_size = 20_000
+        @sub_batch_size = 1000
+        @batch_pause = 0.seconds
+        @sub_batch_pause_ms = 100
+        @batch_max_attempts = 5
+        @throttler = -> { false }
+        @stuck_jobs_timeout = 1.hour
+        @error_handler = ->(error, errored_job) {}
+      end
+
+      def throttler=(value)
+        unless value.respond_to?(:call)
+          raise ArgumentError, "background_migrations throttler must be a callable."
+        end
+
+        @throttler = value
+      end
+    end
+  end
+end