online_migrations 0.14.1 → 0.16.0

data/lib/online_migrations/background_schema_migrations/migration_runner.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module OnlineMigrations
+  module BackgroundSchemaMigrations
+    # Runs single background schema migration.
+    class MigrationRunner
+      attr_reader :migration
+
+      def initialize(migration)
+        @migration = migration
+      end
+
+      def run
+        mark_as_running if migration.enqueued? || migration.failed?
+
+        if migration.composite?
+          migration.children.each do |child_migration|
+            runner = self.class.new(child_migration)
+            runner.run
+          end
+        else
+          do_run
+        end
+      end
+
+      private
+        def mark_as_running
+          Migration.transaction do
+            migration.running!
+            migration.parent.running! if migration.parent
+          end
+        end
+
+        def do_run
+          migration_payload = notifications_payload(migration)
+
+          if migration.attempts == 0
+            ActiveSupport::Notifications.instrument("started.background_schema_migrations", migration_payload)
+          else
+            ActiveSupport::Notifications.instrument("retried.background_schema_migrations", migration_payload)
+          end
+
+          if should_throttle?
+            ActiveSupport::Notifications.instrument("throttled.background_schema_migrations", migration_payload)
+            return
+          end
+
+          migration.update!(
+            attempts: migration.attempts + 1,
+            status: :running,
+            started_at: Time.current,
+            finished_at: nil,
+            error_class: nil,
+            error_message: nil,
+            backtrace: nil
+          )
+
+          ActiveSupport::Notifications.instrument("run.background_schema_migrations", migration_payload) do
+            migration.run
+          end
+
+          migration.update!(status: :succeeded, finished_at: Time.current)
+
+          ActiveSupport::Notifications.instrument("completed.background_schema_migrations", migration_payload)
+
+          complete_parent_if_needed(migration) if migration.parent.present?
+        rescue Exception => e # rubocop:disable Lint/RescueException
+          backtrace_cleaner = ::OnlineMigrations.config.backtrace_cleaner
+
+          migration.update!(
+            status: :failed,
+            finished_at: Time.current,
+            error_class: e.class.name,
+            error_message: e.message,
+            backtrace: backtrace_cleaner ? backtrace_cleaner.clean(e.backtrace) : e.backtrace
+          )
+
+          ::OnlineMigrations.config.background_schema_migrations.error_handler.call(e, migration)
+        end
+
+        def should_throttle?
+          ::OnlineMigrations.config.throttler.call
+        end
+
+        def complete_parent_if_needed(migration)
+          parent = migration.parent
+          completed = false
+
+          parent.with_lock do
+            children = parent.children.select(:status)
+            if children.all?(&:succeeded?)
+              parent.succeeded!
+              completed = true
+            elsif children.any?(&:failed?)
+              parent.failed!
+              completed = true
+            end
+          end
+
+          if completed
+            ActiveSupport::Notifications.instrument("completed.background_migrations", notifications_payload(migration))
+          end
+        end
+
+        def notifications_payload(migration)
+          { background_schema_migration: migration }
+        end
+    end
+  end
+end

data/lib/online_migrations/background_schema_migrations/migration_status_validator.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module OnlineMigrations
+  module BackgroundSchemaMigrations
+    # @private
+    class MigrationStatusValidator < ActiveModel::Validator
+      VALID_STATUS_TRANSITIONS = {
+        # enqueued -> running occurs when the migration starts performing.
+        "enqueued" => ["running"],
+        # running -> succeeded occurs when the migration completes successfully.
+        # running -> failed occurs when the migration raises an exception when running and retry attempts exceeded.
+        "running" => ["succeeded", "failed"],
+        # failed -> enqueued occurs when the failed migration is enqueued to be retried.
+        # failed -> running occurs when the failed migration is retried.
+        "failed" => ["enqueued", "running"],
+      }
+
+      def validate(record)
+        return if !record.status_changed?
+
+        previous_status, new_status = record.status_change
+        valid_new_statuses = VALID_STATUS_TRANSITIONS.fetch(previous_status, [])
+
+        if !valid_new_statuses.include?(new_status)
+          record.errors.add(
+            :status,
+            "cannot transition background schema migration from status #{previous_status} to #{new_status}"
+          )
+        end
+      end
+    end
+  end
+end

data/lib/online_migrations/background_schema_migrations/scheduler.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module OnlineMigrations
+  module BackgroundSchemaMigrations
+    # Class responsible for scheduling background schema migrations.
+    # It selects a single migration and runs it if there is no currently running migration.
+    #
+    # Scheduler should be configured to run periodically, for example, via cron.
+    # @example Run via whenever
+    #   # add this to schedule.rb
+    #   every 1.minute do
+    #     runner "OnlineMigrations.run_background_schema_migrations"
+    #   end
+    #
+    class Scheduler
+      def self.run
+        new.run
+      end
+
+      # Runs Scheduler
+      def run
+        migration = Migration.runnable.enqueued.queue_order.first || Migration.retriable.queue_order.first
+        if migration
+          runner = MigrationRunner.new(migration)
+          runner.run
+        end
+      end
+    end
+  end
+end

data/lib/online_migrations/command_checker.rb

@@ -527,6 +527,30 @@ module OnlineMigrations
       end
       alias add_belongs_to add_reference
 
+      def add_reference_concurrently(table_name, ref_name, **options)
+        # Always added by default in 5.0+
+        index = options.fetch(:index, true)
+
+        if index.is_a?(Hash) && index[:using].to_s == "hash" && postgresql_version < Gem::Version.new("10")
+          raise_error :add_hash_index
+        end
+
+        foreign_key = options.fetch(:foreign_key, false)
+
+        if foreign_key
+          foreign_table_name = Utils.foreign_table_name(ref_name, options)
+          @foreign_key_tables << foreign_table_name.to_s
+        end
+
+        if !options[:polymorphic]
+          type = (options[:type] || :bigint).to_sym
+          column_name = "#{ref_name}_id"
+
+          foreign_key_options = foreign_key.is_a?(Hash) ? foreign_key : {}
+          check_mismatched_foreign_key_type(table_name, column_name, type, **foreign_key_options)
+        end
+      end
+
       def add_index(table_name, column_name, **options)
         if options[:using].to_s == "hash" && postgresql_version < Gem::Version.new("10")
           raise_error :add_hash_index
@@ -830,8 +854,14 @@ module OnlineMigrations
           if connection.table_exists?(foreign_table_name)
             primary_key = options[:primary_key] || connection.primary_key(foreign_table_name)
             primary_key_column = column_for(foreign_table_name, primary_key)
+            return if primary_key_column.nil?
+
+            primary_key_type = primary_key_column.sql_type.to_sym
+            # Having bigint foreign keys is safe and people should
+            # detect integer primary keys via some other tools.
+            return if type == :bigint && primary_key_type == :integer
 
-            if primary_key_column && type != primary_key_column.sql_type.to_sym
+            if type != primary_key_type
               raise_error :mismatched_foreign_key_type,
                 table_name: table_name, column_name: column_name
             end

data/lib/online_migrations/command_recorder.rb

@@ -26,6 +26,7 @@ module OnlineMigrations
       :add_reference_concurrently,
       :change_column_type_in_background,
       :enqueue_background_migration,
+      :remove_background_migration,
 
       # column type change helpers
       :initialize_column_type_change,
@@ -77,6 +78,10 @@ module OnlineMigrations
 
       include StraightReversions
 
+      def invert_add_reference_concurrently(args)
+        [:remove_reference, args]
+      end
+
       def invert_swap_column_names(args)
         table_name, column1, column2 = args
         [:swap_column_names, [table_name, column2, column1]]

data/lib/online_migrations/config.rb

@@ -181,6 +181,26 @@ module OnlineMigrations
     #
     attr_accessor :run_background_migrations_inline
 
+    # Allows to throttle background data or schema migrations based on external signal (e.g. database health)
+    #
+    # It will be called before each run.
+    # If throttled, the current run will be retried next time.
+    #
+    # @return [Proc]
+    #
+    # @example
+    #   OnlineMigrations.config.throttler = -> { DatabaseStatus.unhealthy? }
+    #
+    attr_reader :throttler
+
+    # The Active Support backtrace cleaner that will be used to clean the
+    # backtrace of a background data or schema migration that errors.
+    #
+    # @return [ActiveSupport::BacktraceCleaner, nil] the backtrace cleaner to
+    #   use when cleaning a background migrations's backtrace. Defaults to `Rails.backtrace_cleaner`
+    #
+    attr_accessor :backtrace_cleaner
+
     # Configuration object to configure background migrations
     #
     # @return [BackgroundMigrationsConfig]
@@ -188,6 +208,8 @@ module OnlineMigrations
     #
     attr_reader :background_migrations
 
+    attr_reader :background_schema_migrations
+
     def initialize
       @table_renames = {}
       @column_renames = {}
@@ -202,6 +224,7 @@ module OnlineMigrations
       )
 
       @background_migrations = BackgroundMigrations::Config.new
+      @background_schema_migrations = BackgroundSchemaMigrations::Config.new
 
       @checks = []
       @start_after = 0
@@ -213,6 +236,7 @@ module OnlineMigrations
       @enabled_checks = @error_messages.keys.index_with({})
       @verbose_sql_logs = defined?(Rails.env) && (Rails.env.production? || Rails.env.staging?)
       @run_background_migrations_inline = -> { Utils.developer_env? }
+      @throttler = -> { false }
     end
 
     def lock_retrier=(value)
@@ -223,6 +247,14 @@ module OnlineMigrations
       @small_tables = table_names.map(&:to_s)
     end
 
+    def throttler=(value)
+      if !value.respond_to?(:call)
+        raise ArgumentError, "throttler must be a callable."
+      end
+
+      @throttler = value
+    end
+
     # Enables specific check
     #
     # For the list of available checks look at the `error_messages.rb` file.

data/lib/online_migrations/error_messages.rb

@@ -110,7 +110,7 @@ A safer approach is to:
 1. ignore the column:
 
   class <%= model %> < ApplicationRecord
-    self.ignored_columns = [\"<%= column_name %>\"]
+    self.ignored_columns += [\"<%= column_name %>\"]
   end
 
 2. deploy
@@ -151,7 +151,7 @@ It will use a combination of a VIEW and column aliasing to work with both column
 <% if enumerate_columns_in_select_statements %>
 5. Ignore old column
 
-  self.ignored_columns = [:<%= column_name %>]
+  self.ignored_columns += [:<%= column_name %>]
 
 6. Deploy
 7. Remove the column rename config from step 1
@@ -298,7 +298,7 @@ A safer approach is to:
 1. Ignore the column:
 
   class <%= model %> < ApplicationRecord
-    self.ignored_columns = <%= columns %>
+    self.ignored_columns += <%= columns %>
   end
 
 2. Deploy

data/lib/online_migrations/lock_retrier.rb

@@ -229,11 +229,9 @@ module OnlineMigrations
     end
 
     def lock_timeout(*)
-      0
     end
 
     def delay(*)
-      0
     end
 
     def with_lock_retries

data/lib/online_migrations/schema_statements.rb

@@ -4,6 +4,7 @@ module OnlineMigrations
   module SchemaStatements
     include ChangeColumnTypeHelpers
     include BackgroundMigrations::MigrationHelpers
+    include BackgroundSchemaMigrations::MigrationHelpers
 
     # Updates the value of a column in batches.
     #

data/lib/online_migrations/utils.rb

@@ -135,6 +135,13 @@ module OnlineMigrations
         connection.select_value(query) == "v"
       end
 
+      def find_connection_class(model)
+        model.ancestors.find do |parent|
+          parent == ActiveRecord::Base ||
+            (parent.is_a?(Class) && parent.abstract_class?)
+        end
+      end
+
       def shard_names(model)
         model.ancestors.each do |ancestor|
           # There is no official method to get shard names from the model.
@@ -151,6 +158,11 @@ module OnlineMigrations
         db_config = ActiveRecord::Base.configurations.configs_for(env_name: env)
         db_config.reject(&:replica?).size > 1
       end
+
+      def run_background_migrations_inline?
+        run_inline = OnlineMigrations.config.run_background_migrations_inline
+        run_inline && run_inline.call
+      end
     end
   end
 end

data/lib/online_migrations/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module OnlineMigrations
-  VERSION = "0.14.1"
+  VERSION = "0.16.0"
 end

data/lib/online_migrations.rb

@@ -4,6 +4,7 @@ require "active_record"
 
 require "online_migrations/version"
 require "online_migrations/utils"
+require "online_migrations/background_schema_migrations/migration_helpers"
 require "online_migrations/change_column_type_helpers"
 require "online_migrations/background_migrations/migration_helpers"
 require "online_migrations/schema_statements"
@@ -22,6 +23,7 @@ module OnlineMigrations
 
   extend ActiveSupport::Autoload
 
+  autoload :ApplicationRecord
   autoload :BatchIterator
   autoload :VerboseSqlLogs
   autoload :ForeignKeysCollector
@@ -52,7 +54,6 @@ module OnlineMigrations
     autoload :DeleteOrphanedRecords
     autoload :PerformActionOnRelation
     autoload :ResetCounters
-    autoload :ApplicationRecord
     autoload :MigrationJob
     autoload :Migration
     autoload :MigrationJobRunner
@@ -60,6 +61,16 @@ module OnlineMigrations
     autoload :Scheduler
   end
 
+  module BackgroundSchemaMigrations
+    extend ActiveSupport::Autoload
+
+    autoload :Config
+    autoload :Migration
+    autoload :MigrationStatusValidator
+    autoload :MigrationRunner
+    autoload :Scheduler
+  end
+
   class << self
     # @private
     attr_accessor :current_migration
@@ -72,10 +83,16 @@ module OnlineMigrations
       @config ||= Config.new
     end
 
-    # Run background migrations
+    # Run background data migrations
     def run_background_migrations
       BackgroundMigrations::Scheduler.run
     end
+    alias run_background_data_migrations run_background_migrations
+
+    # Run background schema migrations
+    def run_background_schema_migrations
+      BackgroundSchemaMigrations::Scheduler.run
+    end
 
     def deprecator
       @deprecator ||=

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: online_migrations
 version: !ruby/object:Gem::Version
-  version: 0.14.1
+  version: 0.16.0
 platform: ruby
 authors:
 - fatkodima
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-02-21 00:00:00.000000000 Z
+date: 2024-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -34,18 +34,21 @@ files:
 - CHANGELOG.md
 - LICENSE.txt
 - README.md
-- docs/background_migrations.md
+- docs/background_data_migrations.md
+- docs/background_schema_migrations.md
 - docs/configuring.md
 - lib/generators/online_migrations/background_migration_generator.rb
 - lib/generators/online_migrations/install_generator.rb
 - lib/generators/online_migrations/templates/add_sharding_to_online_migrations.rb.tt
 - lib/generators/online_migrations/templates/background_migration.rb.tt
+- lib/generators/online_migrations/templates/create_background_schema_migrations.rb.tt
 - lib/generators/online_migrations/templates/initializer.rb.tt
+- lib/generators/online_migrations/templates/install_migration.rb.tt
 - lib/generators/online_migrations/templates/migration.rb.tt
 - lib/generators/online_migrations/upgrade_generator.rb
 - lib/online_migrations.rb
+- lib/online_migrations/application_record.rb
 - lib/online_migrations/background_migration.rb
-- lib/online_migrations/background_migrations/application_record.rb
 - lib/online_migrations/background_migrations/backfill_column.rb
 - lib/online_migrations/background_migrations/background_migration_class_validator.rb
 - lib/online_migrations/background_migrations/config.rb
@@ -62,6 +65,12 @@ files:
 - lib/online_migrations/background_migrations/perform_action_on_relation.rb
 - lib/online_migrations/background_migrations/reset_counters.rb
 - lib/online_migrations/background_migrations/scheduler.rb
+- lib/online_migrations/background_schema_migrations/config.rb
+- lib/online_migrations/background_schema_migrations/migration.rb
+- lib/online_migrations/background_schema_migrations/migration_helpers.rb
+- lib/online_migrations/background_schema_migrations/migration_runner.rb
+- lib/online_migrations/background_schema_migrations/migration_status_validator.rb
+- lib/online_migrations/background_schema_migrations/scheduler.rb
 - lib/online_migrations/batch_iterator.rb
 - lib/online_migrations/change_column_type_helpers.rb
 - lib/online_migrations/command_checker.rb

data/lib/online_migrations/background_migrations/application_record.rb

@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-module OnlineMigrations
-  module BackgroundMigrations
-    # Base class for all records used by this gem.
-    #
-    # Can be extended to setup different database where all tables related to
-    # online_migrations will live.
-    class ApplicationRecord < ActiveRecord::Base
-      self.abstract_class = true
-    end
-  end
-end