online_migrations 0.25.0 → 0.27.0

data/lib/online_migrations/change_column_type_helpers.rb

@@ -267,11 +267,7 @@ module OnlineMigrations
         __copy_indexes(table_name, column_name, tmp_column_name)
         __copy_foreign_keys(table_name, column_name, tmp_column_name)
         __copy_check_constraints(table_name, column_name, tmp_column_name)
-
-        # Exclusion constraints were added in https://github.com/rails/rails/pull/40224.
-        if Utils.ar_version >= 7.1
-          __copy_exclusion_constraints(table_name, column_name, tmp_column_name)
-        end
+        __copy_exclusion_constraints(table_name, column_name, tmp_column_name)
 
         if column_name == primary_key
           __finalize_primary_key_type_change(table_name, column_name, column_names)
@@ -358,13 +354,22 @@ module OnlineMigrations
     # @see #cleanup_column_type_change
     #
     def cleanup_columns_type_change(table_name, *column_names)
-      conversions = column_names.index_with do |column_name|
-        __change_type_column(column_name)
+      tmp_column_names = column_names.map { |column_name| __change_type_column(column_name) }
+
+      # Safely remove existing indexes and foreign keys first, if any.
+      tmp_column_names.each do |column_name|
+        __indexes_for(table_name, column_name).each do |index|
+          remove_index(table_name, name: index.name, algorithm: :concurrently)
+        end
+
+        __foreign_keys_for(table_name, column_name).each do |fk|
+          remove_foreign_key(table_name, name: fk.name)
+        end
       end
 
       transaction do
-        __remove_copy_triggers(table_name, conversions.keys, conversions.values)
-        remove_columns(table_name, *conversions.values)
+        __remove_copy_triggers(table_name, column_names, tmp_column_names)
+        remove_columns(table_name, *tmp_column_names)
       end
     end
 
@@ -401,12 +406,13 @@ module OnlineMigrations
         to_column    = to_column.to_s
 
         __indexes_for(table_name, from_column).each do |index|
+          columns = index.columns
           new_columns =
             # Expression index.
-            if index.columns.is_a?(String)
-              index.columns.gsub(/\b#{from_column}\b/, to_column)
+            if columns.is_a?(String)
+              columns.gsub(/\b#{from_column}\b/, to_column)
             else
-              index.columns.map do |column|
+              columns.map do |column|
                 column == from_column ? to_column : column
               end
             end
@@ -429,6 +435,16 @@ module OnlineMigrations
             options[:opclass] = opclasses
           end
 
+          # If the index name is custom - do not rely on auto generated index names, because this
+          # doesn't work (idempotency check does not work, rails does not consider ':where' option)
+          # when there are partial and "classic" indexes on the same columns.
+          if index.name == index_name(table_name, columns)
+            options[:name] = index_name(table_name, new_columns)
+          else
+            truncated_index_name = index.name[0, max_identifier_length - "_2".length]
+            options[:name] = "#{truncated_index_name}_2"
+          end
+
           add_index(table_name, new_columns, **options, algorithm: :concurrently)
         end
       end
@@ -461,10 +477,11 @@ module OnlineMigrations
       end
 
       def __copy_check_constraints(table_name, from_column, to_column)
-        check_constraints = check_constraints(table_name).select { |c| c.expression.include?(from_column) }
+        from_column_re = /\b#{from_column}\b/
+        check_constraints = check_constraints(table_name).select { |c| c.expression.match?(from_column_re) }
 
         check_constraints.each do |check|
-          new_expression = check.expression.gsub(from_column, to_column)
+          new_expression = check.expression.gsub(from_column_re, to_column)
 
           add_check_constraint(table_name, new_expression, validate: false)
 
@@ -511,6 +528,13 @@ module OnlineMigrations
           # Lock the table explicitly to prevent new rows being inserted
           execute("LOCK TABLE #{quoted_table_name} IN ACCESS EXCLUSIVE MODE")
 
+          # https://stackoverflow.com/questions/47301722/how-can-view-depends-on-primary-key-constraint-in-postgres
+          #
+          # PG::DependentObjectsStillExist: ERROR:  cannot drop constraint appointments_pkey on table appointments because other objects depend on it (PG::DependentObjectsStillExist)
+          # DETAIL:  view appointment_statuses depends on constraint appointments_pkey on table appointments
+          # HINT:  Use DROP ... CASCADE to drop the dependent objects too.
+          views = __drop_dependent_views(table_name)
+
           swap_column_names(table_name, column_name, tmp_column_name)
 
           # We need to update the trigger function in order to make PostgreSQL to
@@ -530,6 +554,8 @@ module OnlineMigrations
           execute("ALTER TABLE #{quoted_table_name} DROP CONSTRAINT #{quote_table_name(pkey_constraint_name)}")
           rename_index(table_name, pkey_index_name, pkey_constraint_name)
           execute("ALTER TABLE #{quoted_table_name} ADD CONSTRAINT #{quote_table_name(pkey_constraint_name)} PRIMARY KEY USING INDEX #{quote_table_name(pkey_constraint_name)}")
+
+          __recreate_views(views)
         end
       end
 
@@ -579,5 +605,40 @@ module OnlineMigrations
         function_name = __copy_triggers_name(table_name, column_names, tmp_column_names)
         execute("ALTER FUNCTION #{quote_table_name(function_name)}() RESET ALL")
       end
+
+      # https://stackoverflow.com/questions/69458819/is-there-any-way-to-list-all-the-views-related-to-a-table-in-the-existing-postgr
+      def __drop_dependent_views(table_name)
+        views = select_all(<<~SQL)
+          SELECT
+            u.view_schema AS schema,
+            u.view_name AS name,
+            v.view_definition AS definition,
+            c.relkind = 'm' AS materialized
+          FROM information_schema.view_table_usage u
+            JOIN information_schema.views v ON u.view_schema = v.table_schema
+              AND u.view_name = v.table_name
+            JOIN pg_class c ON c.relname = u.view_name
+          WHERE u.table_schema NOT IN ('information_schema', 'pg_catalog')
+            AND u.table_name = #{quote(table_name)}
+          ORDER BY u.view_schema, u.view_name
+        SQL
+
+        views.each do |row|
+          execute("DROP VIEW #{quote_table_name(row['schema'])}.#{quote_table_name(row['name'])}")
+        end
+
+        views
+      end
+
+      def __recreate_views(views)
+        views.each do |row|
+          schema, name, definition, materialized = row.values_at("schema", "name", "definition", "materialized")
+
+          execute(<<~SQL)
+            CREATE#{' MATERIALIZED' if materialized} VIEW #{quote_table_name(schema)}.#{quote_table_name(name)} AS
+            #{definition}
+          SQL
+        end
+      end
   end
 end

data/lib/online_migrations/command_checker.rb

@@ -80,6 +80,7 @@ module OnlineMigrations
         add_inheritance_column: "adding-a-single-table-inheritance-column",
         mismatched_foreign_key_type: "mismatched-reference-column-types",
       }
+      private_constant :ERROR_MESSAGE_TO_LINK
 
       def check_database_version
         return if defined?(@database_version_checked)
@@ -375,6 +376,31 @@ module OnlineMigrations
               cleanup_code: command_str(:cleanup_column_type_change, table_name, column_name),
               cleanup_down_code: command_str(:initialize_column_type_change, table_name, column_name, existing_type)
           end
+
+          # Constraints must be rechecked.
+          # PostgreSQL recommends dropping constraints before and adding them back.
+          # https://www.postgresql.org/docs/current/ddl-alter.html#DDL-ALTER-COLUMN-TYPE
+          constraints = connection.check_constraints(table_name).select do |c|
+            c.validated? && c.expression.match?(/\b#{column_name}\b/)
+          end
+
+          if constraints.any?
+            change_commands = constraints.map do |c|
+              command_str(:remove_check_constraint, table_name, c.expression, { name: c.name })
+            end
+            change_commands << command_str(:change_column, table_name, column_name, type, **options)
+            constraints.each do |c|
+              change_commands << command_str(:add_check_constraint, table_name, c.expression, { name: c.name, validate: false })
+            end
+
+            validate_commands = constraints.map do |c|
+              command_str(:validate_check_constraint, table_name, { name: c.name })
+            end
+
+            raise_error :change_column_constraint,
+              change_column_code: change_commands.join("\n    "),
+              validate_constraint_code: validate_commands.join("\n    ")
+          end
         end
       end
 
@@ -388,9 +414,9 @@ module OnlineMigrations
         if !allow_null && !new_or_small_table?(table_name)
           # In PostgreSQL 12+ you can add a check constraint to the table
           # and then "promote" it to NOT NULL for the column.
-          safe = check_constraints(table_name).any? do |c|
-            c["def"] == "CHECK ((#{column_name} IS NOT NULL))" ||
-              c["def"] == "CHECK ((#{connection.quote_column_name(column_name)} IS NOT NULL))"
+          safe = connection.check_constraints(table_name).select(&:validated?).any? do |c|
+            c.expression == "#{column_name} IS NOT NULL" ||
+              c.expression == "#{connection.quote_column_name(column_name)} IS NOT NULL"
           end
 
           if !safe
@@ -601,7 +627,7 @@ module OnlineMigrations
       def add_unique_constraint(table_name, column_name = nil, **options)
         return if new_or_small_table?(table_name) || options[:using_index] || !column_name
 
-        index_name = Utils.index_name(table_name, column_name)
+        index_name = connection.index_name(table_name, column_name)
 
         raise_error :add_unique_constraint,
           add_index_code: command_str(:add_index, table_name, column_name, unique: true, name: index_name, algorithm: :concurrently),
@@ -765,18 +791,6 @@ module OnlineMigrations
         "chk_rails_#{hashed_identifier}"
       end
 
-      def check_constraints(table_name)
-        constraints_query = <<~SQL
-          SELECT pg_get_constraintdef(oid) AS def
-          FROM pg_constraint
-          WHERE contype = 'c'
-            AND convalidated
-            AND conrelid = #{connection.quote(table_name)}::regclass
-        SQL
-
-        connection.select_all(constraints_query).to_a
-      end
-
       def check_inheritance_column(table_name, column_name, default)
         if column_name.to_s == ActiveRecord::Base.inheritance_column && !default.nil?
           raise_error :add_inheritance_column,
@@ -834,10 +848,8 @@ module OnlineMigrations
       end
 
       def index_include_column?(index, column)
-        # Expression index
-        (index.columns.is_a?(String) && index.columns.include?(column)) ||
-          index.columns.include?(column) ||
-          (Utils.ar_version >= 7.1 && index.include && index.include.include?(column)) ||
+        index.columns.include?(column) ||
+          (index.include && index.include.include?(column)) ||
           (index.where && index.where.include?(column))
       end
 

data/lib/online_migrations/config.rb

@@ -73,6 +73,13 @@ module OnlineMigrations
       end
     end
 
+    # Whether to require safety reason explanation when calling #safery_assured
+    #
+    # Disabled by default
+    # @return [Boolean]
+    #
+    attr_accessor :require_safety_assured_reason
+
     # Whether to perform checks when migrating down
     #
     # Disabled by default
@@ -203,10 +210,10 @@ module OnlineMigrations
 
     # Configuration object to configure background migrations
     #
-    # @return [BackgroundMigrationsConfig]
-    # @see BackgroundMigrationsConfig
+    # @return [BackgroundDataMigrations::Config]
+    # @see BackgroundDataMigrations::Config
     #
-    attr_reader :background_migrations
+    attr_reader :background_data_migrations
 
     attr_reader :background_schema_migrations
 
@@ -223,13 +230,14 @@ module OnlineMigrations
         lock_timeout: 0.2.seconds
       )
 
-      @background_migrations = BackgroundMigrations::Config.new
+      @background_data_migrations = BackgroundDataMigrations::Config.new
       @background_schema_migrations = BackgroundSchemaMigrations::Config.new
 
       @checks = []
       @start_after = 0
       @target_version = nil
       @small_tables = []
+      @require_safety_assured_reason = false
       @check_down = false
       @auto_analyze = false
       @alphabetize_schema = false

data/lib/online_migrations/data_migration.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+gem "sidekiq", ">= 7.3.3"
+require "sidekiq"
+
+module OnlineMigrations
+  # Base class that is inherited by the host application's data migration classes.
+  class DataMigration
+    class NotFoundError < NameError; end
+
+    class << self
+      # Finds a Data Migration with the given name.
+      #
+      # @param name [String] the name of the Data Migration to be found.
+      #
+      # @return [DataMigration] the Data Migration with the given name.
+      #
+      # @raise [NotFoundError] if a Data Migration with the given name does not exist.
+      #
+      def named(name)
+        namespace = OnlineMigrations.config.background_data_migrations.migrations_module.constantize
+        internal_namespace = ::OnlineMigrations::BackgroundDataMigrations
+
+        migration = "#{namespace}::#{name}".safe_constantize ||
+                    "#{internal_namespace}::#{name}".safe_constantize
+
+        raise NotFoundError.new("Data Migration #{name} not found", name) if migration.nil?
+        if !(migration.is_a?(Class) && migration < self)
+          raise NotFoundError.new("#{name} is not a Data Migration", name)
+        end
+
+        migration
+      end
+
+      # @private
+      attr_accessor :active_record_enumerator_batch_size
+
+      # Limit the number of records that will be fetched in a single query when
+      # iterating over an Active Record collection migration.
+      #
+      # @param size [Integer] the number of records to fetch in a single query.
+      #
+      def collection_batch_size(size)
+        self.active_record_enumerator_batch_size = size
+      end
+    end
+
+    # A hook to override that will be called when the migration starts running.
+    #
+    def after_start
+    end
+
+    # A hook to override that will be called around 'process' each time.
+    #
+    # Can be useful for some metrics collection, performance tracking etc.
+    #
+    def around_process
+      yield
+    end
+
+    # A hook to override that will be called when the migration resumes its work.
+    #
+    def after_resume
+    end
+
+    # A hook to override that will be called each time the migration is interrupted.
+    #
+    # This can be due to interruption or sidekiq stopping.
+    #
+    def after_stop
+    end
+
+    # A hook to override that will be called when the migration finished its work.
+    #
+    def after_complete
+    end
+
+    # A hook to override that will be called when the migration is paused.
+    #
+    def after_pause
+    end
+
+    # A hook to override that will be called when the migration is cancelled.
+    #
+    def after_cancel
+    end
+
+    # The collection to be processed.
+    #
+    # @return [ActiveRecord::Relation, ActiveRecord::Batches::BatchEnumerator, Array, Enumerator]
+    #
+    # @raise [NotImplementedError] with a message advising subclasses to override this method.
+    #
+    def collection
+      raise NotImplementedError, "#{self.class.name} must implement a 'collection' method"
+    end
+
+    # The action to be performed on each item from the collection.
+    #
+    # @param _item the current item from the collection being iterated
+    # @raise [NotImplementedError] with a message advising subclasses to override this method.
+    #
+    def process(_item)
+      raise NotImplementedError, "#{self.class.name} must implement a 'process' method"
+    end
+
+    # Total count of iterations to be performed (optional, to be able to show progress).
+    #
+    # @return [Integer, nil]
+    #
+    def count
+    end
+
+    # Enumerator builder. You may override this method to return any Enumerator yielding
+    # pairs of `[item, item_cursor]`, instead of using `collection`.
+    #
+    # It is useful when it is not practical or impossible to define an explicit collection
+    # in the `collection` method.
+    #
+    # @param cursor [Object, nil] cursor position to resume from, or nil on initial call.
+    #
+    # @return [Enumerator]
+    #
+    def build_enumerator(cursor:)
+    end
+  end
+end

data/lib/online_migrations/error_messages.rb

@@ -242,6 +242,22 @@ during writes works automatically). For most column type changes, this does not
 8. Remove changes from step 3, if any
 9. Deploy",
 
+      change_column_constraint: "Changing the type of a column that has check constraints blocks reads and writes
+while every row is checked. Drop the check constraints on the column before
+changing the type and add them back afterwards.
+
+class <%= migration_name %> < <%= migration_parent %>
+  def change
+    <%= change_column_code %>
+  end
+end
+
+class Validate<%= migration_name %> < <%= migration_parent %>
+  def change
+    <%= validate_constraint_code %>
+  end
+end",
+
       change_column_default:
 "Partial writes are enabled, which can cause incorrect values
 to be inserted when changing the default value of a column.

data/lib/online_migrations/index_definition.rb

@@ -20,7 +20,7 @@ module OnlineMigrations
       # For ActiveRecord::ConnectionAdapters::IndexDefinition is for expression indexes,
       # `columns` is a string
       table == other.table &&
-        (columns & Array(other.columns)).any?
+        columns.intersect?(Array(other.columns))
     end
 
     # @param other [OnlineMigrations::IndexDefinition, ActiveRecord::ConnectionAdapters::IndexDefinition]

data/lib/online_migrations/lock_retrier.rb

@@ -91,10 +91,13 @@ module OnlineMigrations
         else
           yield
         end
-      rescue ActiveRecord::LockWaitTimeout
+      rescue ActiveRecord::LockWaitTimeout, ActiveRecord::Deadlocked => e
         if current_attempt <= attempts
           current_delay = delay(current_attempt)
-          Utils.say("Lock timeout. Retrying in #{current_delay} seconds...")
+
+          problem = e.is_a?(ActiveRecord::Deadlocked) ? "Deadlock detected." : "Lock timeout."
+          Utils.say("#{problem} Retrying in #{current_delay} seconds...")
+
           sleep(current_delay)
           retry
         end

data/lib/online_migrations/migration.rb

@@ -45,7 +45,14 @@ module OnlineMigrations
     # @example
     #   safety_assured { remove_column(:users, :some_column) }
     #
-    def safety_assured(&block)
+    def safety_assured(reason = nil, &block)
+      config = OnlineMigrations.config
+      safe_version = version && version <= config.start_after
+
+      if config.require_safety_assured_reason && reason.blank? && !safe_version
+        raise OnlineMigrations::Error, "Specify a safety reason explanation when calling #safety_assured."
+      end
+
       command_checker.class.safety_assured(&block)
     end
 

data/lib/online_migrations/schema_cache.rb

@@ -3,84 +3,6 @@
 module OnlineMigrations
   # @private
   module SchemaCache
-    def primary_keys(table_name)
-      if (renamed_table = renamed_table?(table_name))
-        super(renamed_table)
-      elsif renamed_column?(table_name)
-        super(column_rename_table(table_name))
-      else
-        super
-      end
-    end
-
-    def columns(table_name)
-      if (renamed_table = renamed_table?(table_name))
-        super(renamed_table)
-      elsif renamed_column?(table_name)
-        columns = super(column_rename_table(table_name))
-        OnlineMigrations.config.column_renames[table_name].each do |old_column_name, new_column_name|
-          duplicate_column(old_column_name, new_column_name, columns)
-        end
-        columns
-      else
-        super.reject { |column| column.name.end_with?("_for_type_change") }
-      end
-    end
-
-    def indexes(table_name)
-      if (renamed_table = renamed_table?(table_name))
-        super(renamed_table)
-      elsif renamed_column?(table_name)
-        super(column_rename_table(table_name))
-      else
-        super
-      end
-    end
-
-    def clear_data_source_cache!(name)
-      if (renamed_table = renamed_table?(name))
-        super(renamed_table)
-      end
-
-      if renamed_column?(name)
-        super(column_rename_table(name))
-      end
-
-      super
-    end
-
-    private
-      def renamed_table?(table_name)
-        table_renames = OnlineMigrations.config.table_renames
-        if table_renames.key?(table_name)
-          views = connection.views
-          table_renames[table_name] if views.include?(table_name)
-        end
-      end
-
-      def renamed_column?(table_name)
-        column_renames = OnlineMigrations.config.column_renames
-        column_renames.key?(table_name) && connection.views.include?(table_name)
-      end
-
-      def column_rename_table(table_name)
-        "#{table_name}_column_rename"
-      end
-
-      def duplicate_column(old_column_name, new_column_name, columns)
-        old_column = columns.find { |column| column.name == old_column_name }
-        new_column = old_column.dup
-        # Active Record defines only reader for :name
-        new_column.instance_variable_set(:@name, new_column_name)
-        # Correspond to the Active Record freezing of each column
-        columns << new_column.freeze
-      end
-  end
-
-  # @private
-  module SchemaCache71
-    # Active Record >= 7.1 changed signature of the methods,
-    # see https://github.com/rails/rails/pull/48716.
     def primary_keys(connection, table_name)
       if (renamed_table = renamed_table?(connection, table_name))
         super(connection, renamed_table)