strong_migrations 2.0.0 → 2.5.0

data/lib/strong_migrations/checker.rb

@@ -11,10 +11,16 @@ module StrongMigrations
 
     def initialize(migration)
       @migration = migration
+      reset
+    end
+
+    def reset
       @new_tables = []
       @new_columns = []
       @timeouts_set = false
       @committed = false
+      @transaction_disabled = false
+      @skip_retries = false
     end
 
     def self.safety_assured
@@ -27,7 +33,10 @@ module StrongMigrations
       end
     end
 
-    def perform(method, *args)
+    def perform(method, *args, &block)
+      return yield if skip?
+
+      check_adapter
       check_version_supported
       set_timeouts
       check_lock_timeout
@@ -70,6 +79,8 @@ module StrongMigrations
           check_remove_index(*args)
         when :rename_column
           check_rename_column
+        when :rename_schema
+          check_rename_schema
         when :rename_table
           check_rename_table
         when :validate_check_constraint
@@ -96,19 +107,26 @@ module StrongMigrations
           # TODO figure out how to handle methods that generate multiple statements
           # like add_reference(table, ref, index: {algorithm: :concurrently})
           # lock timeout after first statement will cause retry to fail
-          retry_lock_timeouts { yield }
+          retry_lock_timeouts { perform_method(method, *args, &block) }
         else
-          yield
+          perform_method(method, *args, &block)
         end
 
       # outdated statistics + a new index can hurt performance of existing queries
-      if StrongMigrations.auto_analyze && direction == :up && method == :add_index
+      if StrongMigrations.auto_analyze && direction == :up && adds_index?(method, *args)
         adapter.analyze_table(args[0])
       end
 
       result
     end
 
+    def perform_method(method, *args)
+      if StrongMigrations.remove_invalid_indexes && direction == :up && method == :add_index && postgresql?
+        remove_invalid_index_if_needed(*args)
+      end
+      yield
+    end
+
     def retry_lock_timeouts(check_committed: false)
       retries = 0
       begin
@@ -129,8 +147,22 @@ module StrongMigrations
       version && version <= StrongMigrations.start_after
     end
 
+    def skip?
+      StrongMigrations.skipped_databases.map(&:to_s).include?(db_config_name)
+    end
+
     private
 
+    def check_adapter
+      return if defined?(@adapter_checked)
+
+      if adapter.instance_of?(Adapters::AbstractAdapter)
+        warn "[strong_migrations] Unsupported adapter: #{connection.adapter_name}. Use StrongMigrations.skip_database(#{db_config_name.to_sym.inspect}) to silence this warning."
+      end
+
+      @adapter_checked = true
+    end
+
     def check_version_supported
       return if defined?(@version_checked)
 
@@ -151,6 +183,9 @@ module StrongMigrations
       if StrongMigrations.statement_timeout
         adapter.set_statement_timeout(StrongMigrations.statement_timeout)
       end
+      if StrongMigrations.transaction_timeout
+        adapter.set_transaction_timeout(StrongMigrations.transaction_timeout)
+      end
       if StrongMigrations.lock_timeout
         adapter.set_lock_timeout(StrongMigrations.lock_timeout)
       end
@@ -200,12 +235,57 @@ module StrongMigrations
       @migration.connection
     end
 
+    def db_config_name
+      connection.pool.db_config.name
+    end
+
     def retry_lock_timeouts?(method)
       (
         StrongMigrations.lock_timeout_retries > 0 &&
         !in_transaction? &&
-        method != :transaction
+        method != :transaction &&
+        !@skip_retries
       )
     end
+
+    def without_retries
+      previous_value = @skip_retries
+      begin
+        @skip_retries = true
+        yield
+      ensure
+        @skip_retries = previous_value
+      end
+    end
+
+    def adds_index?(method, *args)
+      case method
+      when :add_index
+        true
+      when :add_reference, :add_belongs_to
+        options = args.extract_options!
+        !!options.fetch(:index, true)
+      else
+        false
+      end
+    end
+
+    # REINDEX INDEX CONCURRENTLY leaves a new invalid index if it fails, so use remove_index instead
+    def remove_invalid_index_if_needed(*args)
+      options = args.extract_options!
+
+      # ensures has same options as existing index
+      # check args to avoid errors with index_exists?
+      return unless args.size == 2 && connection.index_exists?(*args, **options.merge(valid: false))
+
+      table, columns = args
+      index_name = options.fetch(:name, connection.index_name(table, columns))
+
+      @migration.say("Attempting to remove invalid index")
+      without_retries do
+        # TODO pass index schema for extra safety?
+        @migration.remove_index(table, **{name: index_name}.merge(options.slice(:algorithm)))
+      end
+    end
   end
 end

data/lib/strong_migrations/checks.rb

@@ -10,7 +10,7 @@ module StrongMigrations
       if !new_table?(table)
         if postgresql? && options[:validate] != false
           add_options = options.merge(validate: false)
-          name = options[:name] || @migration.check_constraint_options(table, expression, options)[:name]
+          name = options[:name] || connection.check_constraint_options(table, expression, options)[:name]
           validate_options = {name: name}
 
           if StrongMigrations.safe_by_default
@@ -210,17 +210,43 @@ module StrongMigrations
       end
 
       raise_error :change_column, rewrite_blocks: adapter.rewrite_blocks unless safe
+
+      # constraints must be rechecked
+      # Postgres recommends dropping constraints before and adding them back
+      # https://www.postgresql.org/docs/current/ddl-alter.html#DDL-ALTER-COLUMN-TYPE
+      if postgresql?
+        constraints = adapter.constraints(table, column)
+        if constraints.any?
+          change_commands = []
+          constraints.each do |c|
+            change_commands << command_str(:remove_check_constraint, [table, c.expression, {name: c.name}])
+          end
+          change_commands << command_str(:change_column, args + [options])
+          constraints.each do |c|
+            change_commands << command_str(:add_check_constraint, [table, c.expression, {name: c.name, validate: false}])
+          end
+
+          validate_commands = []
+          constraints.each do |c|
+            validate_commands << command_str(:validate_check_constraint, [table, {name: c.name}])
+          end
+
+          raise_error :change_column_constraint,
+            change_column_code: change_commands.join("\n    "),
+            validate_constraint_code: validate_commands.join("\n    ")
+        end
+      end
     end
 
     def check_change_column_default(*args)
       table, column, _default_or_changes = args
 
       # just check ActiveRecord::Base, even though can override on model
-      partial_inserts = ar_version >= 7 ? ActiveRecord::Base.partial_inserts : ActiveRecord::Base.partial_writes
+      partial_inserts = ActiveRecord::Base.partial_inserts
 
       if partial_inserts && !new_column?(table, column)
         raise_error :change_column_default,
-          config: ar_version >= 7 ? "partial_inserts" : "partial_writes"
+          config: "partial_inserts"
       end
     end
 
@@ -228,32 +254,40 @@ module StrongMigrations
       table, column, null, default = args
       if !null
         if postgresql?
-          safe = adapter.constraints(table).any? { |c| c["def"] == "CHECK ((#{column} IS NOT NULL))" || c["def"] == "CHECK ((#{connection.quote_column_name(column)} IS NOT NULL))" }
+          constraints = connection.check_constraints(table)
+          safe = constraints.any? { |c| c.options[:validate] && (c.expression == "#{column} IS NOT NULL" || c.expression == "#{connection.quote_column_name(column)} IS NOT NULL") }
 
           unless safe
+            expression = "#{quote_column_if_needed(column)} IS NOT NULL"
+
             # match https://github.com/nullobject/rein
             constraint_name = "#{table}_#{column}_null"
+            if adapter.max_constraint_name_length && constraint_name.bytesize > adapter.max_constraint_name_length
+              constraint_name = connection.check_constraint_options(table, expression, {})[:name]
 
-            add_code = constraint_str("ALTER TABLE %s ADD CONSTRAINT %s CHECK (%s IS NOT NULL) NOT VALID", [table, constraint_name, column])
-            validate_code = constraint_str("ALTER TABLE %s VALIDATE CONSTRAINT %s", [table, constraint_name])
-            remove_code = constraint_str("ALTER TABLE %s DROP CONSTRAINT %s", [table, constraint_name])
-
-            validate_constraint_code = String.new(command_str(:validate_check_constraint, [table, {name: constraint_name}]))
+              # avoid collision with Active Record naming for safe_by_default
+              if StrongMigrations.safe_by_default
+                constraint_name = constraint_name.sub("rails", "strong_migrations")
+              end
+            end
 
+            add_args = [table, expression, {name: constraint_name, validate: false}]
+            validate_args = [table, {name: constraint_name}]
             change_args = [table, column, null]
-
-            validate_constraint_code << "\n    #{command_str(:change_column_null, change_args)}"
-            validate_constraint_code << "\n    #{command_str(:remove_check_constraint, [table, {name: constraint_name}])}"
+            remove_args = [table, {name: constraint_name}]
 
             if StrongMigrations.safe_by_default
-              safe_change_column_null(add_code, validate_code, change_args, remove_code, default)
+              safe_change_column_null(add_args, validate_args, change_args, remove_args, table, column, default, constraints)
               throw :safe
             end
 
-            add_constraint_code = command_str(:add_check_constraint, [table, "#{quote_column_if_needed(column)} IS NOT NULL", {name: constraint_name, validate: false}])
+            add_constraint_code = command_str(:add_check_constraint, add_args)
 
+            up_code = String.new(command_str(:validate_check_constraint, validate_args))
+            up_code << "\n    #{command_str(:change_column_null, change_args)}"
+            up_code << "\n    #{command_str(:remove_check_constraint, remove_args)}"
             down_code = "#{add_constraint_code}\n    #{command_str(:change_column_null, [table, column, true])}"
-            validate_constraint_code = "def up\n    #{validate_constraint_code}\n  end\n\n  def down\n    #{down_code}\n  end"
+            validate_constraint_code = "def up\n    #{up_code}\n  end\n\n  def down\n    #{down_code}\n  end"
 
             raise_error :change_column_null_postgresql,
               add_constraint_code: add_constraint_code,
@@ -302,29 +336,28 @@ module StrongMigrations
       columns =
         case method
         when :remove_timestamps
-          ["created_at", "updated_at"]
+          [:created_at, :updated_at]
         when :remove_column
-          [args[1].to_s]
+          [args[1]]
         when :remove_columns
-          # Active Record 6.1+ supports options
           if args.last.is_a?(Hash)
-            args[1..-2].map(&:to_s)
+            args[1..-2]
           else
-            args[1..-1].map(&:to_s)
+            args[1..-1]
           end
         else
           options = args[2] || {}
           reference = args[1]
           cols = []
-          cols << "#{reference}_type" if options[:polymorphic]
-          cols << "#{reference}_id"
+          cols << "#{reference}_type".to_sym if options[:polymorphic]
+          cols << "#{reference}_id".to_sym
           cols
         end
 
-      code = "self.ignored_columns += #{columns.inspect}"
+      code = "self.ignored_columns += #{columns.map(&:to_s).inspect}"
 
       raise_error :remove_column,
-        model: args[0].to_s.classify,
+        model: model_name(args[0]),
         code: code,
         command: command_str(method, args),
         column_suffix: columns.size > 1 ? "s" : ""
@@ -351,6 +384,10 @@ module StrongMigrations
       raise_error :rename_column
     end
 
+    def check_rename_schema
+      raise_error :rename_schema
+    end
+
     def check_rename_table
       raise_error :rename_table
     end
@@ -392,7 +429,7 @@ module StrongMigrations
       message = message + append if append
 
       vars[:migration_name] = @migration.class.name
-      vars[:migration_suffix] = "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
+      vars[:migration_suffix] = migration_suffix
       vars[:base_model] = "ApplicationRecord"
 
       # escape % not followed by {
@@ -433,13 +470,13 @@ module StrongMigrations
     end
 
     def backfill_code(table, column, default, function = false)
-      model = table.to_s.classify
+      model = model_name(table)
       if function
         # update_all(column: Arel.sql(default)) also works in newer versions of Active Record
         update_expr = "#{quote_column_if_needed(column)} = #{default}"
-        "#{model}.unscoped.in_batches do |relation| \n      relation.where(#{column}: nil).update_all(#{update_expr.inspect})\n      sleep(0.01)\n    end"
+        "#{model}.unscoped.in_batches(of: 10000) do |relation| \n      relation.where(#{column}: nil).update_all(#{update_expr.inspect})\n      sleep(0.01)\n    end"
       else
-        "#{model}.unscoped.in_batches do |relation| \n      relation.update_all #{column}: #{default.inspect}\n      sleep(0.01)\n    end"
+        "#{model}.unscoped.in_batches(of: 10000) do |relation| \n      relation.where(#{column}: nil).update_all #{column}: #{default.inspect}\n      sleep(0.01)\n    end"
       end
     end
 
@@ -456,5 +493,13 @@ module StrongMigrations
     def new_column?(table, column)
       new_table?(table) || @new_columns.include?([table.to_s, column.to_s])
     end
+
+    def migration_suffix
+      "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
+    end
+
+    def model_name(table)
+      table.to_s.classify
+    end
   end
 end

data/lib/strong_migrations/error_messages.rb

@@ -60,6 +60,22 @@ while the entire table is rewritten. A safer approach is to:
     change_column_with_not_null:
 "Changing the type is safe, but setting NOT NULL is not.",
 
+    change_column_constraint: "Changing the type of a column that has check constraints blocks reads and writes
+while every row is checked. Drop the check constraints on the column before
+changing the type and add them back afterwards.
+
+class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  def change
+    %{change_column_code}
+  end
+end
+
+class Validate%{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  def change
+    %{validate_constraint_code}
+  end
+end",
+
     remove_column: "Active Record caches attributes, which causes problems
 when removing columns. Be sure to ignore the column%{column_suffix}:
 
@@ -86,6 +102,17 @@ in your application. A safer approach is to:
 5. Stop writing to the old column
 6. Drop the old column",
 
+    rename_schema:
+"Renaming a schema that's in use will cause errors
+in your application. A safer approach is to:
+
+1. Create a new schema
+2. Write to both schemas
+3. Backfill data from the old schema to the new schema
+4. Move reads from the old schema to the new schema
+5. Stop writing to the old schema
+6. Drop the old schema",
+
     rename_table:
 "Renaming a table that's in use will cause errors
 in your application. A safer approach is to:
@@ -147,7 +174,7 @@ you're doing is safe before proceeding, then wrap it in a safety_assured { ... }
     create_table:
 "The force option will destroy existing tables.
 If this is intended, drop the existing table first.
-Otherwise, remove the force option.",
+In any case, remove the force option.",
 
     execute:
 "Strong Migrations does not support inspecting what happens inside an

data/lib/strong_migrations/migration.rb

@@ -7,10 +7,7 @@ module StrongMigrations
     end
 
     def method_missing(method, *args)
-      return super if is_a?(ActiveRecord::Schema)
-
-      # Active Record 7.0.2+ versioned schema
-      return super if defined?(ActiveRecord::Schema::Definition) && is_a?(ActiveRecord::Schema::Definition)
+      return super if is_a?(ActiveRecord::Schema) || is_a?(ActiveRecord::Schema::Definition)
 
       catch(:safe) do
         strong_migrations_checker.perform(method, *args) do
@@ -18,7 +15,8 @@ module StrongMigrations
         end
       end
     end
-    ruby2_keywords(:method_missing) if respond_to?(:ruby2_keywords, true)
+    # same as ActiveRecord::Migration
+    ruby2_keywords(:method_missing)
 
     def revert(*)
       if strong_migrations_checker.version_safe?

data/lib/strong_migrations/{database_tasks.rb → migration_context.rb}

@@ -1,9 +1,26 @@
 module StrongMigrations
-  module DatabaseTasks
-    # Active Record 7 adds version argument
-    def migrate(*args)
+  module MigrationContext
+    def up(...)
       super
     rescue => e
+      strong_migrations_process_exception(e)
+    end
+
+    def down(...)
+      super
+    rescue => e
+      strong_migrations_process_exception(e)
+    end
+
+    def run(...)
+      super
+    rescue => e
+      strong_migrations_process_exception(e)
+    end
+
+    private
+
+    def strong_migrations_process_exception(e)
       if e.cause.is_a?(StrongMigrations::Error)
         # strip cause and clean backtrace
         def e.cause

data/lib/strong_migrations/migrator.rb

@@ -1,18 +1,20 @@
 module StrongMigrations
   module Migrator
-    def ddl_transaction(migration, *args)
+    def ddl_transaction(migration, ...)
       return super unless StrongMigrations.lock_timeout_retries > 0 && use_transaction?(migration)
 
       # handle MigrationProxy class
-      migration = migration.send(:migration) if migration.respond_to?(:migration, true)
+      migration = migration.send(:migration) if !migration.is_a?(ActiveRecord::Migration) && migration.respond_to?(:migration, true)
 
-      # retry migration since the entire transaction needs to be rerun
       checker = migration.send(:strong_migrations_checker)
+      return super if checker.skip?
+
+      # retry migration since the entire transaction needs to be rerun
       checker.retry_lock_timeouts(check_committed: true) do
         # failed transaction reverts timeout, so need to re-apply
-        checker.timeouts_set = false
+        checker.reset
 
-        super(migration, *args)
+        super(migration, ...)
       end
     end
   end

data/lib/strong_migrations/safe_methods.rb

@@ -1,10 +1,9 @@
 module StrongMigrations
   module SafeMethods
     def safe_by_default_method?(method)
-      StrongMigrations.safe_by_default && [:add_index, :add_belongs_to, :add_reference, :remove_index, :add_foreign_key, :add_check_constraint, :change_column_null].include?(method)
+      StrongMigrations.safe_by_default && !version_safe? && [:add_index, :add_belongs_to, :add_reference, :remove_index, :add_foreign_key, :add_check_constraint, :change_column_null].include?(method)
     end
 
-    # TODO check if invalid index with expected name exists and remove if needed
     def safe_add_index(*args, **options)
       disable_transaction
       @migration.add_index(*args, **options.merge(algorithm: :concurrently))
@@ -47,14 +46,14 @@ module StrongMigrations
     def safe_add_foreign_key(from_table, to_table, *args, **options)
       @migration.reversible do |dir|
         dir.up do
-          @migration.add_foreign_key(from_table, to_table, *args, **options.merge(validate: false))
+          if !connection.foreign_key_exists?(from_table, to_table, **options.merge(validate: false))
+            @migration.add_foreign_key(from_table, to_table, *args, **options.merge(validate: false))
+          end
           disable_transaction
-          validate_options = options.slice(:column, :name)
-          @migration.validate_foreign_key(from_table, to_table, **validate_options)
+          @migration.validate_foreign_key(from_table, to_table, **options.slice(:column, :name))
         end
         dir.down do
-          remove_options = options.slice(:column, :name)
-          @migration.remove_foreign_key(from_table, to_table, **remove_options)
+          @migration.remove_foreign_key(from_table, to_table, **options.slice(:column, :name))
         end
       end
     end
@@ -62,7 +61,10 @@ module StrongMigrations
     def safe_add_check_constraint(table, expression, *args, add_options, validate_options)
       @migration.reversible do |dir|
         dir.up do
-          @migration.add_check_constraint(table, expression, *args, **add_options)
+          # only skip invalid constraints
+          unless connection.check_constraints(table).any? { |c| c.options[:name] == validate_options[:name] && !c.options[:validate] }
+            @migration.add_check_constraint(table, expression, *args, **add_options)
+          end
           disable_transaction
           @migration.validate_check_constraint(table, **validate_options)
         end
@@ -72,35 +74,60 @@ module StrongMigrations
       end
     end
 
-    def safe_change_column_null(add_code, validate_code, change_args, remove_code, default)
+    def safe_change_column_null(add_args, validate_args, change_args, remove_args, table, column, default, constraints)
       @migration.reversible do |dir|
         dir.up do
           unless default.nil?
-            raise Error, "default value not supported yet with safe_by_default"
-          end
+            # TODO search for parent model if needed
+            if connection.pool != ActiveRecord::Base.connection_pool
+              raise_error :change_column_null,
+                code: backfill_code(table, column, default)
+            end
+
+            model =
+              Class.new(ActiveRecord::Base) do
+                self.table_name = table
+
+                def self.to_s
+                  "Backfill"
+                end
+              end
+
+            update_sql =
+              model.connection_pool.with_connection do |c|
+                quoted_column = c.quote_column_name(column)
+                quoted_default = c.quote_default_expression(default, c.send(:column_for, table, column))
+                "#{quoted_column} = #{quoted_default}"
+              end
 
-          @migration.safety_assured do
-            @migration.execute(add_code)
+            @migration.say("Backfilling default")
             disable_transaction
-            @migration.execute(validate_code)
-          end
-          if change_args
-            @migration.change_column_null(*change_args)
-            @migration.safety_assured do
-              @migration.execute(remove_code)
+            model.unscoped.in_batches(of: 10000) do |relation|
+              relation.where(column => nil).update_all(update_sql)
+              sleep(0.01)
             end
           end
+
+          add_options = add_args.extract_options!
+          validate_options = validate_args.extract_options!
+          remove_options = remove_args.extract_options!
+
+          # only skip invalid constraints
+          unless constraints.any? { |c| c.options[:name] == validate_options[:name] && !c.options[:validate] }
+            @migration.add_check_constraint(*add_args, **add_options)
+          end
+          disable_transaction
+
+          connection.begin_db_transaction
+          @migration.validate_check_constraint(*validate_args, **validate_options)
+          @migration.change_column_null(*change_args)
+          @migration.remove_check_constraint(*remove_args, **remove_options)
+          connection.commit_db_transaction
         end
         dir.down do
-          if change_args
-            down_args = change_args.dup
-            down_args[2] = true
-            @migration.change_column_null(*down_args)
-          else
-            @migration.safety_assured do
-              @migration.execute(remove_code)
-            end
-          end
+          down_args = change_args.dup
+          down_args[2] = true
+          @migration.change_column_null(*down_args)
         end
       end
     end
@@ -109,13 +136,13 @@ module StrongMigrations
     # so just commit at start
     def disable_transaction
       if in_transaction? && !transaction_disabled
-        @migration.connection.commit_db_transaction
+        connection.commit_db_transaction
         self.transaction_disabled = true
       end
     end
 
     def in_transaction?
-      @migration.connection.open_transactions > 0
+      connection.open_transactions > 0
     end
   end
 end

data/lib/strong_migrations/schema_dumper.rb

@@ -1,9 +1,9 @@
 module StrongMigrations
   module SchemaDumper
-    def initialize(connection, *args, **options)
+    def initialize(connection, ...)
       return super unless StrongMigrations.alphabetize_schema
 
-      super(WrappedConnection.new(connection), *args, **options)
+      super(WrappedConnection.new(connection), ...)
     end
   end
 
@@ -14,8 +14,19 @@ module StrongMigrations
       @connection = connection
     end
 
-    def columns(*args, **options)
-      @connection.columns(*args, **options).sort_by(&:name)
+    def columns(...)
+      @connection.columns(...).sort_by(&:name)
+    end
+
+    # forward private methods with send
+    # method_missing cannot tell how method was called
+    # this is not ideal, but other solutions have drawbacks
+    def send(name, ...)
+      if respond_to?(name, true)
+        super
+      else
+        @connection.send(name, ...)
+      end
     end
   end
 end

data/lib/strong_migrations/version.rb

@@ -1,3 +1,3 @@
 module StrongMigrations
-  VERSION = "2.0.0"
+  VERSION = "2.5.0"
 end