strong_migrations 0.6.0 → 2.3.0

data/lib/strong_migrations/error_messages.rb

@@ -0,0 +1,276 @@
+module StrongMigrations
+  self.error_messages = {
+    add_column_default:
+"Adding a column with a %{default_type} default blocks %{rewrite_blocks} while the entire table is rewritten.
+Instead, add the column without a default value, then change the default.
+
+class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  def up
+    %{add_command}
+    %{change_command}
+  end
+
+  def down
+    %{remove_command}
+  end
+end
+
+Then backfill the existing rows in the Rails console or a separate migration with disable_ddl_transaction!.
+
+class Backfill%{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  disable_ddl_transaction!
+
+  def up
+    %{code}
+  end
+end",
+
+    add_column_default_callable:
+"Strong Migrations does not support inspecting callable default values.
+Please make really sure you're not calling a VOLATILE function,
+then wrap it in a safety_assured { ... } block.",
+
+    add_column_json:
+"There's no equality operator for the json column type, which can cause errors for
+existing SELECT DISTINCT queries in your application. Use jsonb instead.
+
+class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  def change
+    %{command}
+  end
+end",
+
+    add_column_generated_stored:
+"Adding a stored generated column blocks %{rewrite_blocks} while the entire table is rewritten.",
+
+    add_column_auto_incrementing:
+"Adding an auto-incrementing column blocks %{rewrite_blocks} while the entire table is rewritten.",
+
+    change_column:
+"Changing the type of an existing column blocks %{rewrite_blocks}
+while the entire table is rewritten. A safer approach is to:
+
+1. Create a new column
+2. Write to both columns
+3. Backfill data from the old column to the new column
+4. Move reads from the old column to the new column
+5. Stop writing to the old column
+6. Drop the old column",
+
+    change_column_with_not_null:
+"Changing the type is safe, but setting NOT NULL is not.",
+
+    change_column_constraint: "Changing the type of a column that has check constraints blocks reads and writes
+while every row is checked. Drop the check constraints on the column before
+changing the type and add them back afterwards.
+
+class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  def change
+    %{change_column_code}
+  end
+end
+
+class Validate%{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  def change
+    %{validate_constraint_code}
+  end
+end",
+
+    remove_column: "Active Record caches attributes, which causes problems
+when removing columns. Be sure to ignore the column%{column_suffix}:
+
+class %{model} < %{base_model}
+  %{code}
+end
+
+Deploy the code, then wrap this step in a safety_assured { ... } block.
+
+class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  def change
+    safety_assured { %{command} }
+  end
+end",
+
+    rename_column:
+"Renaming a column that's in use will cause errors
+in your application. A safer approach is to:
+
+1. Create a new column
+2. Write to both columns
+3. Backfill data from the old column to the new column
+4. Move reads from the old column to the new column
+5. Stop writing to the old column
+6. Drop the old column",
+
+    rename_table:
+"Renaming a table that's in use will cause errors
+in your application. A safer approach is to:
+
+1. Create a new table. Don't forget to recreate indexes from the old table
+2. Write to both tables
+3. Backfill data from the old table to the new table
+4. Move reads from the old table to the new table
+5. Stop writing to the old table
+6. Drop the old table",
+
+    add_reference:
+"%{headline} Instead, use:
+
+class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  disable_ddl_transaction!
+
+  def change
+    %{command}
+  end
+end",
+
+    add_index:
+"Adding an index non-concurrently blocks writes. Instead, use:
+
+class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  disable_ddl_transaction!
+
+  def change
+    %{command}
+  end
+end",
+
+    remove_index:
+"Removing an index non-concurrently blocks writes. Instead, use:
+
+class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  disable_ddl_transaction!
+
+  def change
+    %{command}
+  end
+end",
+
+    add_index_columns:
+"Adding a non-unique index with more than three columns rarely improves performance.
+Instead, start an index with columns that narrow down the results the most.",
+
+    add_index_corruption:
+"Adding an index concurrently can cause silent data corruption in Postgres 14.0 to 14.3.
+Upgrade Postgres before adding new indexes, or wrap this step in a safety_assured { ... } block
+to accept the risk.",
+
+    change_table:
+"Strong Migrations does not support inspecting what happens inside a
+change_table block, so cannot help you here. Please make really sure that what
+you're doing is safe before proceeding, then wrap it in a safety_assured { ... } block.",
+
+    create_table:
+"The force option will destroy existing tables.
+If this is intended, drop the existing table first.
+In any case, remove the force option.",
+
+    execute:
+"Strong Migrations does not support inspecting what happens inside an
+execute call, so cannot help you here. Please make really sure that what
+you're doing is safe before proceeding, then wrap it in a safety_assured { ... } block.",
+
+    change_column_default:
+"Partial writes are enabled, which can cause incorrect values
+to be inserted when changing the default value of a column.
+Disable partial writes in config/application.rb:
+
+config.active_record.%{config} = false",
+
+    change_column_null:
+"Passing a default value to change_column_null runs a single UPDATE query,
+which can cause downtime. Instead, backfill the existing rows in the
+Rails console or a separate migration with disable_ddl_transaction!.
+
+class Backfill%{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  disable_ddl_transaction!
+
+  def up
+    %{code}
+  end
+end",
+
+    change_column_null_postgresql:
+"Setting NOT NULL on an existing column blocks reads and writes while every row is checked.
+Instead, add a check constraint and validate it in a separate migration.
+
+class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  def change
+    %{add_constraint_code}
+  end
+end
+
+class Validate%{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  %{validate_constraint_code}
+end",
+
+    change_column_null_mysql:
+"Setting NOT NULL on an existing column is not safe without strict mode enabled.",
+
+    add_foreign_key:
+"Adding a foreign key blocks writes on both tables. Instead,
+add the foreign key without validating existing rows,
+then validate them in a separate migration.
+
+class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  def change
+    %{add_foreign_key_code}
+  end
+end
+
+class Validate%{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  def change
+    %{validate_foreign_key_code}
+  end
+end",
+
+    validate_foreign_key:
+"Validating a foreign key while writes are blocked is dangerous.
+Use disable_ddl_transaction! or a separate migration.",
+
+    add_check_constraint:
+"Adding a check constraint key blocks reads and writes while every row is checked.
+Instead, add the check constraint without validating existing rows,
+then validate them in a separate migration.
+
+class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  def change
+    %{add_check_constraint_code}
+  end
+end
+
+class Validate%{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  def change
+    %{validate_check_constraint_code}
+  end
+end",
+
+    add_check_constraint_mysql:
+"Adding a check constraint to an existing table is not safe with your database engine.",
+
+    validate_check_constraint:
+"Validating a check constraint while writes are blocked is dangerous.
+Use disable_ddl_transaction! or a separate migration.",
+
+    add_exclusion_constraint:
+"Adding an exclusion constraint blocks reads and writes while every row is checked.",
+
+    add_unique_constraint:
+"Adding a unique constraint creates a unique index, which blocks reads and writes.
+Instead, create a unique index concurrently, then use it for the constraint.
+
+class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  disable_ddl_transaction!
+
+  def up
+    %{index_command}
+    %{constraint_command}
+  end
+
+  def down
+    %{remove_command}
+  end
+end"
+  }
+  self.enabled_checks = (error_messages.keys - [:remove_index]).map { |k| [k, {}] }.to_h
+end

data/lib/strong_migrations/migration.rb

@@ -3,16 +3,34 @@ module StrongMigrations
     def migrate(direction)
       strong_migrations_checker.direction = direction
       super
+      connection.begin_db_transaction if strong_migrations_checker.transaction_disabled
     end
 
     def method_missing(method, *args)
-      strong_migrations_checker.perform(method, *args) do
+      return super if is_a?(ActiveRecord::Schema)
+
+      # Active Record 7.0.2+ versioned schema
+      return super if defined?(ActiveRecord::Schema::Definition) && is_a?(ActiveRecord::Schema::Definition)
+
+      catch(:safe) do
+        strong_migrations_checker.perform(method, *args) do
+          super
+        end
+      end
+    end
+    # same as ActiveRecord::Migration
+    ruby2_keywords(:method_missing)
+
+    def revert(*)
+      if strong_migrations_checker.version_safe?
+        safety_assured { super }
+      else
         super
       end
     end
 
     def safety_assured
-      strong_migrations_checker.safety_assured do
+      strong_migrations_checker.class.safety_assured do
         yield
       end
     end

data/lib/strong_migrations/{database_tasks.rb → migration_context.rb}

@@ -1,8 +1,26 @@
 module StrongMigrations
-  module DatabaseTasks
-    def migrate
+  module MigrationContext
+    def up(...)
       super
     rescue => e
+      strong_migrations_process_exception(e)
+    end
+
+    def down(...)
+      super
+    rescue => e
+      strong_migrations_process_exception(e)
+    end
+
+    def run(...)
+      super
+    rescue => e
+      strong_migrations_process_exception(e)
+    end
+
+    private
+
+    def strong_migrations_process_exception(e)
       if e.cause.is_a?(StrongMigrations::Error)
         # strip cause and clean backtrace
         def e.cause

data/lib/strong_migrations/migrator.rb

@@ -0,0 +1,21 @@
+module StrongMigrations
+  module Migrator
+    def ddl_transaction(migration, ...)
+      return super unless StrongMigrations.lock_timeout_retries > 0 && use_transaction?(migration)
+
+      # handle MigrationProxy class
+      migration = migration.send(:migration) if migration.respond_to?(:migration, true)
+
+      checker = migration.send(:strong_migrations_checker)
+      return super if checker.skip?
+
+      # retry migration since the entire transaction needs to be rerun
+      checker.retry_lock_timeouts(check_committed: true) do
+        # failed transaction reverts timeout, so need to re-apply
+        checker.reset
+
+        super(migration, ...)
+      end
+    end
+  end
+end

data/lib/strong_migrations/railtie.rb

@@ -5,10 +5,6 @@ module StrongMigrations
   class Railtie < Rails::Railtie
     rake_tasks do
       load "tasks/strong_migrations.rake"
-
-      ["db:drop", "db:reset", "db:schema:load", "db:structure:load"].each do |t|
-        Rake::Task[t].enhance ["strong_migrations:safety_assured"]
-      end
     end
   end
 end

data/lib/strong_migrations/safe_methods.rb

@@ -0,0 +1,150 @@
+module StrongMigrations
+  module SafeMethods
+    def safe_by_default_method?(method)
+      StrongMigrations.safe_by_default && !version_safe? && [:add_index, :add_belongs_to, :add_reference, :remove_index, :add_foreign_key, :add_check_constraint, :change_column_null].include?(method)
+    end
+
+    def safe_add_index(*args, **options)
+      disable_transaction
+      @migration.add_index(*args, **options.merge(algorithm: :concurrently))
+    end
+
+    def safe_remove_index(*args, **options)
+      disable_transaction
+      @migration.remove_index(*args, **options.merge(algorithm: :concurrently))
+    end
+
+    def safe_add_reference(table, reference, *args, **options)
+      @migration.reversible do |dir|
+        dir.up do
+          disable_transaction
+          foreign_key = options.delete(:foreign_key)
+          @migration.add_reference(table, reference, *args, **options)
+          if foreign_key
+            # same as Active Record
+            name =
+              if foreign_key.is_a?(Hash) && foreign_key[:to_table]
+                foreign_key[:to_table]
+              else
+                (ActiveRecord::Base.pluralize_table_names ? reference.to_s.pluralize : reference).to_sym
+              end
+
+            foreign_key_opts = foreign_key.is_a?(Hash) ? foreign_key.except(:to_table) : {}
+            if reference
+              @migration.add_foreign_key(table, name, column: "#{reference}_id", **foreign_key_opts)
+            else
+              @migration.add_foreign_key(table, name, **foreign_key_opts)
+            end
+          end
+        end
+        dir.down do
+          @migration.remove_reference(table, reference)
+        end
+      end
+    end
+
+    def safe_add_foreign_key(from_table, to_table, *args, **options)
+      @migration.reversible do |dir|
+        dir.up do
+          # validate option is unintentionally ignored for Active Record < 7.1
+          # https://github.com/rails/rails/pull/45896
+          if !connection.foreign_key_exists?(from_table, to_table, **options.merge(validate: false))
+            @migration.add_foreign_key(from_table, to_table, *args, **options.merge(validate: false))
+          end
+          disable_transaction
+          @migration.validate_foreign_key(from_table, to_table, **options.slice(:column, :name))
+        end
+        dir.down do
+          @migration.remove_foreign_key(from_table, to_table, **options.slice(:column, :name))
+        end
+      end
+    end
+
+    def safe_add_check_constraint(table, expression, *args, add_options, validate_options)
+      @migration.reversible do |dir|
+        dir.up do
+          # only skip invalid constraints
+          unless connection.check_constraints(table).any? { |c| c.options[:name] == validate_options[:name] && !c.options[:validate] }
+            @migration.add_check_constraint(table, expression, *args, **add_options)
+          end
+          disable_transaction
+          @migration.validate_check_constraint(table, **validate_options)
+        end
+        dir.down do
+          @migration.remove_check_constraint(table, expression, **add_options.except(:validate))
+        end
+      end
+    end
+
+    def safe_change_column_null(add_args, validate_args, change_args, remove_args, table, column, default, constraints)
+      @migration.reversible do |dir|
+        dir.up do
+          unless default.nil?
+            # TODO search for parent model if needed
+            if connection.pool != ActiveRecord::Base.connection_pool
+              raise_error :change_column_null,
+                code: backfill_code(table, column, default)
+            end
+
+            model =
+              Class.new(ActiveRecord::Base) do
+                self.table_name = table
+
+                def self.to_s
+                  "Backfill"
+                end
+              end
+
+            update_sql =
+              model.connection_pool.with_connection do |c|
+                quoted_column = c.quote_column_name(column)
+                quoted_default = c.quote_default_expression(default, c.send(:column_for, table, column))
+                "#{quoted_column} = #{quoted_default}"
+              end
+
+            @migration.say("Backfilling default")
+            disable_transaction
+            model.unscoped.in_batches(of: 10000) do |relation|
+              relation.where(column => nil).update_all(update_sql)
+              sleep(0.01)
+            end
+          end
+
+          add_options = add_args.extract_options!
+          validate_options = validate_args.extract_options!
+          remove_options = remove_args.extract_options!
+
+          # only skip invalid constraints
+          unless constraints.any? { |c| c.options[:name] == validate_options[:name] && !c.options[:validate] }
+            @migration.add_check_constraint(*add_args, **add_options)
+          end
+          disable_transaction
+
+          connection.begin_db_transaction
+          @migration.validate_check_constraint(*validate_args, **validate_options)
+          @migration.change_column_null(*change_args)
+          @migration.remove_check_constraint(*remove_args, **remove_options)
+          connection.commit_db_transaction
+        end
+        dir.down do
+          down_args = change_args.dup
+          down_args[2] = true
+          @migration.change_column_null(*down_args)
+        end
+      end
+    end
+
+    # hard to commit at right time when reverting
+    # so just commit at start
+    def disable_transaction
+      if in_transaction? && !transaction_disabled
+        connection.commit_db_transaction
+        self.transaction_disabled = true
+      end
+    end
+
+    def in_transaction?
+      connection.open_transactions > 0
+    end
+  end
+end

data/lib/strong_migrations/schema_dumper.rb

@@ -0,0 +1,32 @@
+module StrongMigrations
+  module SchemaDumper
+    def initialize(connection, ...)
+      return super unless StrongMigrations.alphabetize_schema
+
+      super(WrappedConnection.new(connection), ...)
+    end
+  end
+
+  class WrappedConnection
+    delegate_missing_to :@connection
+
+    def initialize(connection)
+      @connection = connection
+    end
+
+    def columns(...)
+      @connection.columns(...).sort_by(&:name)
+    end
+
+    # forward private methods with send
+    # method_missing cannot tell how method was called
+    # this is not ideal, but other solutions have drawbacks
+    def send(name, ...)
+      if respond_to?(name, true)
+        super
+      else
+        @connection.send(name, ...)
+      end
+    end
+  end
+end

data/lib/strong_migrations/version.rb

@@ -1,3 +1,3 @@
 module StrongMigrations
-  VERSION = "0.6.0"
+  VERSION = "2.3.0"
 end