online_migrations 0.9.2 → 0.11.0

data/lib/online_migrations/background_migrations/migration_job.rb

@@ -2,7 +2,7 @@
 
 module OnlineMigrations
   module BackgroundMigrations
-    class MigrationJob < ActiveRecord::Base
+    class MigrationJob < ApplicationRecord
       STATUSES = [
         :enqueued,
         :running,
@@ -12,12 +12,11 @@ module OnlineMigrations
 
       self.table_name = :background_migration_jobs
 
-      # For Active Record <= 4.2 needs to fully specify enum values
-      scope :active, -> { where(status: [statuses[:enqueued], statuses[:running]]) }
-      scope :completed, -> { where(status: [statuses[:failed], statuses[:succeeded]]) }
+      scope :active, -> { where(status: [:enqueued, :running]) }
+      scope :completed, -> { where(status: [:failed, :succeeded]) }
       scope :stuck, -> do
-        timeout = ::OnlineMigrations.config.background_migrations.stuck_jobs_timeout
-        active.where("updated_at <= ?", timeout.ago)
+        timeout = OnlineMigrations.config.background_migrations.stuck_jobs_timeout
+        active.where("updated_at <= ?", timeout.seconds.ago)
       end
 
       scope :retriable, -> do
@@ -26,7 +25,7 @@ module OnlineMigrations
         stuck_sql             = connection.unprepared_statement { stuck.to_sql }
         failed_retriable_sql  = connection.unprepared_statement { failed_retriable.to_sql }
 
-        from(Arel.sql(<<-SQL.strip_heredoc))
+        from(Arel.sql(<<~SQL))
           (
             (#{failed_retriable_sql})
             UNION
@@ -35,19 +34,16 @@ module OnlineMigrations
         SQL
       end
 
-      scope :except_succeeded, -> { where("status != ?", statuses[:succeeded]) }
+      scope :except_succeeded, -> { where.not(status: :succeeded) }
       scope :attempts_exceeded, -> { where("attempts >= max_attempts") }
 
-      enum status: STATUSES.map { |status| [status, status.to_s] }.to_h
+      enum status: STATUSES.index_with(&:to_s)
 
       delegate :migration_class, :migration_object, :migration_relation, :batch_column_name,
         :arguments, :batch_pause, to: :migration
 
       belongs_to :migration
 
-      # For Active Record 5.0+ this is validated by default from belongs_to
-      validates :migration, presence: true
-
       validates :min_value, :max_value, presence: true, numericality: { greater_than: 0 }
       validate :values_in_migration_range, if: :min_value?
       validate :validate_values_order, if: :min_value?
@@ -56,6 +52,13 @@ module OnlineMigrations
 
       before_create :copy_settings_from_migration
 
+      # Whether the job is considered stuck (is running for some configured time).
+      #
+      def stuck?
+        timeout = OnlineMigrations.config.background_migrations.stuck_jobs_timeout
+        running? && updated_at <= timeout.seconds.ago
+      end
+
       # Mark this job as ready to be processed again.
       #
       # This is used when retrying failed jobs.

data/lib/online_migrations/background_migrations/migration_job_runner.rb

@@ -6,7 +6,7 @@ module OnlineMigrations
     class MigrationJobRunner
       attr_reader :migration_job
 
-      delegate :attempts, :migration_relation, :migration_object, :sub_batch_size,
+      delegate :migration, :attempts, :migration_relation, :migration_object, :sub_batch_size,
         :batch_column_name, :min_value, :max_value, :pause_ms, to: :migration_job
 
       def initialize(migration_job)
@@ -30,7 +30,7 @@ module OnlineMigrations
         )
 
         ActiveSupport::Notifications.instrument("process_batch.background_migrations", job_payload) do
-          run_batch
+          migration.on_shard { run_batch }
         end
 
         migration_job.update!(status: :succeeded, finished_at: Time.current)

data/lib/online_migrations/background_migrations/migration_runner.rb

@@ -12,8 +12,13 @@ module OnlineMigrations
 
       # Runs one background migration job.
       def run_migration_job
-        migration.running! if migration.enqueued?
-        migration_payload = { background_migration: migration }
+        raise "Should not be called on a composite (with sharding) migration" if migration.composite?
+
+        if migration.enqueued?
+          migration.running!
+          migration.parent.running! if migration.parent && migration.parent.enqueued?
+        end
+        migration_payload = notifications_payload(migration)
 
         if !migration.migration_jobs.exists?
           ActiveSupport::Notifications.instrument("started.background_migrations", migration_payload)
@@ -37,6 +42,8 @@ module OnlineMigrations
           end
 
           ActiveSupport::Notifications.instrument("completed.background_migrations", migration_payload)
+
+          complete_parent_if_needed(migration) if migration.parent.present?
         end
 
         next_migration_job
@@ -52,8 +59,15 @@ module OnlineMigrations
 
         migration.running!
 
-        while migration.running?
-          run_migration_job
+        if migration.composite?
+          migration.children.each do |child_migration|
+            runner = self.class.new(child_migration)
+            runner.run_all_migration_jobs
+          end
+        else
+          while migration.running?
+            run_migration_job
+          end
         end
       end
 
@@ -64,13 +78,20 @@ module OnlineMigrations
       def finish
         return if migration.completed?
 
-        # Mark is as finishing to avoid being picked up
-        # by the background migrations scheduler.
-        migration.finishing!
-        migration.reset_failed_jobs_attempts
-
-        while migration.finishing?
-          run_migration_job
+        if migration.composite?
+          migration.children.each do |child_migration|
+            runner = self.class.new(child_migration)
+            runner.finish
+          end
+        else
+          # Mark is as finishing to avoid being picked up
+          # by the background migrations scheduler.
+          migration.finishing!
+          migration.reset_failed_jobs_attempts
+
+          while migration.finishing?
+            run_migration_job
+          end
         end
       end
 
@@ -95,6 +116,30 @@ module OnlineMigrations
             max_value: max_value
           )
         end
+
+        def complete_parent_if_needed(migration)
+          parent = migration.parent
+          completed = false
+
+          parent.with_lock do
+            children = parent.children.select(:status)
+            if children.all?(&:succeeded?)
+              parent.succeeded!
+              completed = true
+            elsif children.any?(&:failed?)
+              parent.failed!
+              completed = true
+            end
+          end
+
+          if completed
+            ActiveSupport::Notifications.instrument("completed.background_migrations", notifications_payload(migration))
+          end
+        end
+
+        def notifications_payload(migration)
+          { background_migration: migration }
+        end
     end
   end
 end

data/lib/online_migrations/background_migrations/reset_counters.rb

@@ -26,7 +26,7 @@ module OnlineMigrations
           counter_name = reflection.counter_cache_column
 
           quoted_association_table = connection.quote_table_name(has_many_association.table_name)
-          count_subquery = <<-SQL.strip_heredoc
+          count_subquery = <<~SQL
             SELECT COUNT(*)
             FROM #{quoted_association_table}
             WHERE #{quoted_association_table}.#{connection.quote_column_name(foreign_key)} =
@@ -41,8 +41,7 @@ module OnlineMigrations
           names = Array.wrap(names)
           options = names.extract_options!
           touch_updates = touch_attributes_with_time(*names, **options)
-          # In Active Record 4.2 sanitize_sql_for_assignment is protected
-          updates << model.send(:sanitize_sql_for_assignment, touch_updates)
+          updates << model.sanitize_sql_for_assignment(touch_updates)
         end
 
         relation.update_all(updates.join(", "))
@@ -64,11 +63,6 @@ module OnlineMigrations
 
             has_many_association = has_many.find do |association|
               counter_cache_column = association.counter_cache_column
-
-              # Active Record <= 4.2 is able to return only explicitly provided `counter_cache` column.
-              if !counter_cache_column && Utils.ar_version <= 4.2
-                counter_cache_column = "#{association.name}_count"
-              end
               counter_cache_column && counter_cache_column.to_sym == counter_association.to_sym
             end
 
@@ -86,7 +80,7 @@ module OnlineMigrations
         def touch_attributes_with_time(*names, time: nil)
           attribute_names = timestamp_attributes_for_update & model.column_names
           attribute_names |= names.map(&:to_s)
-          attribute_names.map { |attribute_name| [attribute_name, time || Time.current] }.to_h
+          attribute_names.index_with(time || Time.current)
         end
 
         def timestamp_attributes_for_update

data/lib/online_migrations/background_migrations/scheduler.rb

@@ -4,15 +4,14 @@ module OnlineMigrations
   module BackgroundMigrations
     # Class responsible for scheduling background migrations.
     # It selects runnable background migrations and runs them one step (one batch) at a time.
-    # A migration is considered runnable if it is not completed and time the interval between
+    # A migration is considered runnable if it is not completed and the time interval between
     #   successive runs has passed.
-    # Scheduler ensures (via advisory locks) that at most one background migration at a time is running per database.
     #
-    # Scheduler should be run via some kind of periodical means, for example, cron.
+    # Scheduler should be configured to run periodically, for example, via cron.
     # @example Run via whenever
     #   # add this to schedule.rb
     #   every 1.minute do
-    #     runner "OnlineMigrations::BackgroundMigrations::Scheduler.run"
+    #     runner "OnlineMigrations.run_background_migrations"
     #   end
     #
     class Scheduler
@@ -22,24 +21,15 @@ module OnlineMigrations
 
       # Runs Scheduler
       def run
-        active_migrations = Migration.active.queue_order
+        active_migrations = Migration.runnable.active.queue_order
         runnable_migrations = active_migrations.select(&:interval_elapsed?)
 
         runnable_migrations.each do |migration|
-          connection = migration.migration_relation.connection
-
-          with_exclusive_lock(connection) do
-            run_migration_job(migration)
-          end
+          run_migration_job(migration)
         end
       end
 
       private
-        def with_exclusive_lock(connection, &block)
-          lock = AdvisoryLock.new(name: "online_migrations_scheduler", connection: connection)
-          lock.with_lock(&block)
-        end
-
         def run_migration_job(migration)
           runner = MigrationRunner.new(migration)
           runner.run_migration_job

data/lib/online_migrations/change_column_type_helpers.rb

@@ -72,6 +72,9 @@ module OnlineMigrations
     #
     # @example With additional column options
     #   initialize_column_type_change(:users, :name, :string, limit: 64)
+    # @example With type casting
+    #   initialize_column_type_change(:users, :settings, :jsonb, type_cast_function: "jsonb")
+    #   initialize_column_type_change(:users, :company_id, :integer, type_cast_function: Arel.sql("company_id::integer"))
     #
     def initialize_column_type_change(table_name, column_name, new_type, **options)
       initialize_columns_type_change(table_name, [[column_name, new_type]], column_name => options)
@@ -91,13 +94,13 @@ module OnlineMigrations
     # @see #initialize_column_type_change
     #
     def initialize_columns_type_change(table_name, columns_and_types, **options)
-      if !columns_and_types.is_a?(Array) || !columns_and_types.all? { |e| e.is_a?(Array) }
+      if !columns_and_types.is_a?(Array) || !columns_and_types.all?(Array)
         raise ArgumentError, "columns_and_types must be an array of arrays"
       end
 
-      conversions = columns_and_types.map do |(column_name, _new_type)|
+      conversions = columns_and_types.to_h do |(column_name, _new_type)|
         [column_name, __change_type_column(column_name)]
-      end.to_h
+      end
 
       if (extra_keys = (options.keys - conversions.keys)).any?
         raise ArgumentError, "Options has unknown keys: #{extra_keys.map(&:inspect).join(', ')}. " \
@@ -105,10 +108,14 @@ module OnlineMigrations
       end
 
       transaction do
+        type_cast_functions = {}.with_indifferent_access
+
         columns_and_types.each do |(column_name, new_type)|
-          old_col = __column_for(table_name, column_name)
+          old_col = column_for(table_name, column_name)
           old_col_options = __options_from_column(old_col, [:collation, :comment])
           column_options = options[column_name] || {}
+          type_cast_function = column_options.delete(:type_cast_function)
+          type_cast_functions[column_name] = type_cast_function if type_cast_function
           tmp_column_name = conversions[column_name]
 
           if raw_connection.server_version >= 11_00_00
@@ -132,7 +139,7 @@ module OnlineMigrations
           end
         end
 
-        __create_copy_triggers(table_name, conversions.keys, conversions.values)
+        __create_copy_triggers(table_name, conversions.keys, conversions.values, type_cast_functions: type_cast_functions)
       end
     end
 
@@ -239,13 +246,15 @@ module OnlineMigrations
     def finalize_columns_type_change(table_name, *column_names)
       __ensure_not_in_transaction!
 
-      conversions = column_names.map do |column_name|
+      conversions = column_names.to_h do |column_name|
         [column_name.to_s, __change_type_column(column_name)]
-      end.to_h
+      end
+
+      primary_key = primary_key(table_name)
 
       conversions.each do |column_name, tmp_column_name|
-        old_column = __column_for(table_name, column_name)
-        column = __column_for(table_name, tmp_column_name)
+        old_column = column_for(table_name, column_name)
+        column = column_for(table_name, tmp_column_name)
 
         # We already set default and NOT NULL for to-be-PK columns
         # for PG >= 11, so can skip this case
@@ -264,7 +273,12 @@ module OnlineMigrations
         __copy_foreign_keys(table_name, column_name, tmp_column_name)
         __copy_check_constraints(table_name, column_name, tmp_column_name)
 
-        if primary_key(table_name) == column_name
+        # Exclusion constraints were added in https://github.com/rails/rails/pull/40224.
+        if Utils.ar_version >= 7.1
+          __copy_exclusion_constraints(table_name, column_name, tmp_column_name)
+        end
+
+        if column_name == primary_key
           __finalize_primary_key_type_change(table_name, column_name, column_names)
         end
       end
@@ -274,7 +288,7 @@ module OnlineMigrations
       # already were swapped and which were not.
       transaction do
         conversions
-          .reject { |column_name, _tmp_column_name| column_name == primary_key(table_name) }
+          .reject { |column_name, _tmp_column_name| column_name == primary_key }
           .each do |column_name, tmp_column_name|
             swap_column_names(table_name, column_name, tmp_column_name)
           end
@@ -302,40 +316,27 @@ module OnlineMigrations
     def revert_finalize_columns_type_change(table_name, *column_names)
       __ensure_not_in_transaction!
 
-      conversions = column_names.map do |column_name|
+      conversions = column_names.to_h do |column_name|
         [column_name.to_s, __change_type_column(column_name)]
-      end.to_h
-
-      transaction do
-        conversions
-          .reject { |column_name, _tmp_column_name| column_name == primary_key(table_name) }
-          .each do |column_name, tmp_column_name|
-            swap_column_names(table_name, column_name, tmp_column_name)
-          end
-
-        __reset_trigger_function(table_name, column_names)
       end
 
-      conversions.each do |column_name, tmp_column_name|
-        indexes(table_name).each do |index|
-          if index.columns.include?(tmp_column_name)
-            remove_index(table_name, tmp_column_name, algorithm: :concurrently)
-          end
-        end
+      primary_key = primary_key(table_name)
+      primary_key_conversion = conversions.delete(primary_key)
 
-        foreign_keys(table_name).each do |fk|
-          if fk.column == tmp_column_name
-            remove_foreign_key(table_name, column: tmp_column_name)
+      # No need to remove indexes, foreign keys etc, because it  can take a significant amount
+      # of time and will be automatically removed if decided to remove the column itself.
+      if conversions.any?
+        transaction do
+          conversions.each do |column_name, tmp_column_name|
+            swap_column_names(table_name, column_name, tmp_column_name)
           end
-        end
 
-        __check_constraints_for(table_name, tmp_column_name).each do |constraint|
-          remove_check_constraint(table_name, name: constraint.constraint_name)
+          __reset_trigger_function(table_name, column_names)
         end
+      end
 
-        if primary_key(table_name) == column_name
-          __finalize_primary_key_type_change(table_name, column_name, column_names)
-        end
+      if primary_key_conversion
+        __finalize_primary_key_type_change(table_name, primary_key, column_names)
       end
     end
 
@@ -362,9 +363,9 @@ module OnlineMigrations
     # @see #cleanup_column_type_change
     #
     def cleanup_columns_type_change(table_name, *column_names)
-      conversions = column_names.map do |column_name|
-        [column_name, __change_type_column(column_name)]
-      end.to_h
+      conversions = column_names.index_with do |column_name|
+        __change_type_column(column_name)
+      end
 
       transaction do
         __remove_copy_triggers(table_name, conversions.keys, conversions.values)
@@ -392,8 +393,8 @@ module OnlineMigrations
         CopyTrigger.on_table(table_name, connection: self).name(from_column, to_column)
       end
 
-      def __create_copy_triggers(table_name, from_column, to_column)
-        CopyTrigger.on_table(table_name, connection: self).create(from_column, to_column)
+      def __create_copy_triggers(table_name, from_columns, to_columns, type_cast_functions: nil)
+        CopyTrigger.on_table(table_name, connection: self).create(from_columns, to_columns, type_cast_functions: type_cast_functions)
       end
 
       def __remove_copy_triggers(table_name, from_column, to_column)
@@ -415,13 +416,11 @@ module OnlineMigrations
               end
             end
 
-          # This is necessary as we can't properly rename indexes such as "taggings_idx".
-          if !index.name.include?(from_column)
-            raise "The index #{index.name} can not be copied as it does not " \
-                  "mention the old column. You have to rename this index manually first."
+          if index.name.include?(from_column)
+            name = index.name.gsub(from_column, to_column)
           end
 
-          name = index.name.gsub(from_column, to_column)
+          name = index_name(table_name, new_columns) if !name || name.length > max_identifier_length
 
           options = {
             unique: index.unique,
@@ -433,8 +432,7 @@ module OnlineMigrations
           options[:using] = index.using if index.using
           options[:where] = index.where if index.where
 
-          # Opclasses were added in 5.2
-          if Utils.ar_version >= 5.2 && !index.opclasses.blank?
+          if index.opclasses.present?
             opclasses = index.opclasses.dup
 
             # Copy the operator classes for the old column (if any) to the new column.
@@ -475,23 +473,39 @@ module OnlineMigrations
       end
 
       def __copy_check_constraints(table_name, from_column, to_column)
-        __check_constraints_for(table_name, from_column).each do |check|
-          expression = check["constraint_def"][/CHECK \({2}(.+)\){2}/, 1]
-          new_expression = expression.gsub(from_column.to_s, to_column.to_s)
+        check_constraints = check_constraints(table_name).select { |c| c.expression.include?(from_column) }
+
+        check_constraints.each do |check|
+          new_expression = check.expression.gsub(from_column, to_column)
 
           add_check_constraint(table_name, new_expression, validate: false)
 
-          if check["valid"]
+          if check.validated?
             validate_check_constraint(table_name, expression: new_expression)
           end
         end
       end
 
+      def __copy_exclusion_constraints(table_name, from_column, to_column)
+        exclusion_constraints = exclusion_constraints(table_name).select { |c| c.expression.include?(from_column) }
+
+        exclusion_constraints.each do |constraint|
+          new_expression = constraint.expression.gsub(from_column, to_column)
+          add_exclusion_constraint(
+            table_name,
+            new_expression,
+            using: constraint.using,
+            where: constraint.where,
+            deferrable: constraint.deferrable
+          )
+        end
+      end
+
       def __set_not_null(table_name, column_name)
         # For PG >= 12 we can "promote" CHECK constraint to NOT NULL constraint:
         # https://github.com/postgres/postgres/commit/bbb96c3704c041d139181c6601e5bc770e045d26
         if raw_connection.server_version >= 12_00_00
-          execute(<<-SQL.strip_heredoc)
+          execute(<<~SQL)
             ALTER TABLE #{quote_table_name(table_name)}
             ALTER #{quote_column_name(column_name)}
             SET NOT NULL
@@ -501,7 +515,7 @@ module OnlineMigrations
           # through PG internal tables.
           # In-depth analysis of implications of this was made, so this approach
           # is considered safe - https://habr.com/ru/company/haulmont/blog/493954/  (in russian).
-          execute(<<-SQL.strip_heredoc)
+          execute(<<~SQL)
             UPDATE pg_catalog.pg_attribute
             SET attnotnull = true
             WHERE attrelid = #{quote(table_name)}::regclass
@@ -510,37 +524,8 @@ module OnlineMigrations
         end
       end
 
-      def __check_constraints_for(table_name, column_name)
-        __check_constraints(table_name).select { |c| c["column_name"] == column_name }
-      end
-
-      def __check_constraints(table_name)
-        schema = __schema_for_table(table_name)
-
-        check_sql = <<-SQL.strip_heredoc
-          SELECT
-            ccu.column_name as column_name,
-            con.conname as constraint_name,
-            pg_get_constraintdef(con.oid) as constraint_def,
-            con.convalidated AS valid
-          FROM pg_catalog.pg_constraint con
-            INNER JOIN pg_catalog.pg_class rel
-              ON rel.oid = con.conrelid
-            INNER JOIN pg_catalog.pg_namespace nsp
-              ON nsp.oid = con.connamespace
-            INNER JOIN information_schema.constraint_column_usage ccu
-              ON con.conname = ccu.constraint_name
-                AND rel.relname = ccu.table_name
-          WHERE rel.relname = #{quote(table_name)}
-            AND con.contype = 'c'
-            AND nsp.nspname = #{schema}
-        SQL
-
-        select_all(check_sql)
-      end
-
       def __rename_constraint(table_name, old_name, new_name)
-        execute(<<-SQL.strip_heredoc)
+        execute(<<~SQL)
           ALTER TABLE #{quote_table_name(table_name)}
           RENAME CONSTRAINT #{quote_column_name(old_name)} TO #{quote_column_name(new_name)}
         SQL
@@ -612,7 +597,7 @@ module OnlineMigrations
       def __referencing_table_names(table_name)
         schema = __schema_for_table(table_name)
 
-        select_values(<<-SQL.strip_heredoc)
+        select_values(<<~SQL)
           SELECT DISTINCT con.conrelid::regclass::text AS conrelname
           FROM pg_catalog.pg_constraint con
             INNER JOIN pg_catalog.pg_namespace nsp