strong_migrations 0.8.0 → 1.0.0

data/lib/strong_migrations/checks.rb

@@ -0,0 +1,402 @@
+# TODO better pattern
+module StrongMigrations
+  module Checks
+    private
+
+    def check_add_check_constraint(args)
+      table, expression, options = args
+      options ||= {}
+
+      if !new_table?(table)
+        if postgresql? && options[:validate] != false
+          add_options = options.merge(validate: false)
+          name = options[:name] || @migration.check_constraint_options(table, expression, options)[:name]
+          validate_options = {name: name}
+
+          if StrongMigrations.safe_by_default
+            safe_add_check_constraint(table, expression, add_options, validate_options)
+            throw :safe
+          end
+
+          raise_error :add_check_constraint,
+            add_check_constraint_code: command_str("add_check_constraint", [table, expression, add_options]),
+            validate_check_constraint_code: command_str("validate_check_constraint", [table, validate_options])
+        elsif mysql? || mariadb?
+          raise_error :add_check_constraint_mysql
+        end
+      end
+    end
+
+    def check_add_column(args)
+      table, column, type, options = args
+      options ||= {}
+      default = options[:default]
+
+      if !default.nil? && !adapter.add_column_default_safe?
+        if options[:null] == false
+          options = options.except(:null)
+          append = "
+
+Then add the NOT NULL constraint in separate migrations."
+        end
+
+        raise_error :add_column_default,
+          add_command: command_str("add_column", [table, column, type, options.except(:default)]),
+          change_command: command_str("change_column_default", [table, column, default]),
+          remove_command: command_str("remove_column", [table, column]),
+          code: backfill_code(table, column, default),
+          append: append,
+          rewrite_blocks: adapter.rewrite_blocks
+      elsif default.is_a?(Proc) && postgresql?
+        # adding a column with a VOLATILE default is not safe
+        # https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-NOTES
+        # functions like random() and clock_timestamp() are VOLATILE
+        # check for Proc to match Active Record
+        raise_error :add_column_default_callable
+      end
+
+      if type.to_s == "json" && postgresql?
+        raise_error :add_column_json,
+          command: command_str("add_column", [table, column, :jsonb, options])
+      end
+    end
+
+    # unlike add_index, we don't make an exception here for new tables
+    #
+    # with add_index, it's fine to lock a new table even after inserting data
+    # since the table won't be in use by the application
+    #
+    # with add_foreign_key, this would cause issues since it locks the referenced table
+    #
+    # it's okay to allow if the table is empty, but not a fan of data-dependent checks,
+    # since the data in production could be different from development
+    #
+    # note: adding foreign_keys with create_table is fine
+    # since the table is always guaranteed to be empty
+    def check_add_foreign_key(args)
+      from_table, to_table, options = args
+      options ||= {}
+
+      validate = options.fetch(:validate, true)
+      if postgresql? && validate
+        if StrongMigrations.safe_by_default
+          safe_add_foreign_key(from_table, to_table, options)
+          throw :safe
+        end
+
+        raise_error :add_foreign_key,
+          add_foreign_key_code: command_str("add_foreign_key", [from_table, to_table, options.merge(validate: false)]),
+          validate_foreign_key_code: command_str("validate_foreign_key", [from_table, to_table])
+      end
+    end
+
+    def check_add_index(args)
+      table, columns, options = args
+      options ||= {}
+
+      if columns.is_a?(Array) && columns.size > 3 && !options[:unique]
+        raise_error :add_index_columns, header: "Best practice"
+      end
+
+      # safe to add non-concurrently to new tables (even after inserting data)
+      # since the table won't be in use by the application
+      if postgresql? && options[:algorithm] != :concurrently && !new_table?(table)
+        if StrongMigrations.safe_by_default
+          safe_add_index(table, columns, options)
+          throw :safe
+        end
+
+        raise_error :add_index, command: command_str("add_index", [table, columns, options.merge(algorithm: :concurrently)])
+      end
+    end
+
+    def check_add_reference(method, args)
+      table, reference, options = args
+      options ||= {}
+
+      if postgresql?
+        index_value = options.fetch(:index, true)
+        concurrently_set = index_value.is_a?(Hash) && index_value[:algorithm] == :concurrently
+        bad_index = index_value && !concurrently_set
+
+        if bad_index || options[:foreign_key]
+          if index_value.is_a?(Hash)
+            options[:index] = options[:index].merge(algorithm: :concurrently)
+          else
+            options = options.merge(index: {algorithm: :concurrently})
+          end
+
+          if StrongMigrations.safe_by_default
+            safe_add_reference(table, reference, options)
+            throw :safe
+          end
+
+          if options.delete(:foreign_key)
+            headline = "Adding a foreign key blocks writes on both tables."
+            append = "
+
+Then add the foreign key in separate migrations."
+          else
+            headline = "Adding an index non-concurrently locks the table."
+          end
+
+          raise_error :add_reference,
+            headline: headline,
+            command: command_str(method, [table, reference, options]),
+            append: append
+        end
+      end
+    end
+
+    def check_change_column(args)
+      table, column, type, options = args
+      options ||= {}
+
+      safe = false
+      existing_column = connection.columns(table).find { |c| c.name.to_s == column.to_s }
+      if existing_column
+        existing_type = existing_column.sql_type.sub(/\(\d+(,\d+)?\)/, "")
+        safe = adapter.change_type_safe?(table, column, type, options, existing_column, existing_type)
+      end
+
+      # unsafe to set NOT NULL for safe types with Postgres
+      # TODO check if safe for MySQL and MariaDB
+      if safe && existing_column.null && options[:null] == false
+        raise_error :change_column_with_not_null
+      end
+
+      raise_error :change_column, rewrite_blocks: adapter.rewrite_blocks unless safe
+    end
+
+    def check_change_column_null(args)
+      table, column, null, default = args
+      if !null
+        if postgresql?
+          safe = false
+          safe_with_check_constraint = adapter.server_version >= Gem::Version.new("12")
+          if safe_with_check_constraint
+            safe = adapter.constraints(table).any? { |c| c["def"] == "CHECK ((#{column} IS NOT NULL))" || c["def"] == "CHECK ((#{connection.quote_column_name(column)} IS NOT NULL))" }
+          end
+
+          unless safe
+            # match https://github.com/nullobject/rein
+            constraint_name = "#{table}_#{column}_null"
+
+            add_code = constraint_str("ALTER TABLE %s ADD CONSTRAINT %s CHECK (%s IS NOT NULL) NOT VALID", [table, constraint_name, column])
+            validate_code = constraint_str("ALTER TABLE %s VALIDATE CONSTRAINT %s", [table, constraint_name])
+            remove_code = constraint_str("ALTER TABLE %s DROP CONSTRAINT %s", [table, constraint_name])
+
+            constraint_methods = ar_version >= 6.1
+
+            validate_constraint_code =
+              if constraint_methods
+                String.new(command_str(:validate_check_constraint, [table, {name: constraint_name}]))
+              else
+                String.new(safety_assured_str(validate_code))
+              end
+
+            if safe_with_check_constraint
+              change_args = [table, column, null]
+
+              validate_constraint_code << "\n    #{command_str(:change_column_null, change_args)}"
+
+              if constraint_methods
+                validate_constraint_code << "\n    #{command_str(:remove_check_constraint, [table, {name: constraint_name}])}"
+              else
+                validate_constraint_code << "\n    #{safety_assured_str(remove_code)}"
+              end
+            end
+
+            if StrongMigrations.safe_by_default
+              safe_change_column_null(add_code, validate_code, change_args, remove_code, default)
+              throw :safe
+            end
+
+            add_constraint_code =
+              if constraint_methods
+                # only quote when needed
+                expr_column = column.to_s =~ /\A[a-z0-9_]+\z/ ? column : connection.quote_column_name(column)
+                command_str(:add_check_constraint, [table, "#{expr_column} IS NOT NULL", {name: constraint_name, validate: false}])
+              else
+                safety_assured_str(add_code)
+              end
+
+            raise_error :change_column_null_postgresql,
+              add_constraint_code: add_constraint_code,
+              validate_constraint_code: "def change\n    #{validate_constraint_code}\n  end"
+          end
+        elsif mysql? || mariadb?
+          unless adapter.strict_mode?
+            raise_error :change_column_null_mysql
+          end
+        end
+
+        if !default.nil?
+          raise_error :change_column_null,
+            code: backfill_code(table, column, default)
+        end
+      end
+    end
+
+    def check_change_table
+      raise_error :change_table, header: "Possibly dangerous operation"
+    end
+
+    def check_create_table(args)
+      table, options = args
+      options ||= {}
+
+      raise_error :create_table if options[:force]
+
+      # keep track of new tables of add_index check
+      @new_tables << table.to_s
+    end
+
+    def check_execute
+      raise_error :execute, header: "Possibly dangerous operation"
+    end
+
+    def check_remove_column(method, args)
+      columns =
+        case method
+        when :remove_timestamps
+          ["created_at", "updated_at"]
+        when :remove_column
+          [args[1].to_s]
+        when :remove_columns
+          # Active Record 6.1+ supports options
+          if args.last.is_a?(Hash)
+            args[1..-2].map(&:to_s)
+          else
+            args[1..-1].map(&:to_s)
+          end
+        else
+          options = args[2] || {}
+          reference = args[1]
+          cols = []
+          cols << "#{reference}_type" if options[:polymorphic]
+          cols << "#{reference}_id"
+          cols
+        end
+
+      code = "self.ignored_columns = #{columns.inspect}"
+
+      raise_error :remove_column,
+        model: args[0].to_s.classify,
+        code: code,
+        command: command_str(method, args),
+        column_suffix: columns.size > 1 ? "s" : ""
+    end
+
+    def check_remove_index(args)
+      table, options = args
+      unless options.is_a?(Hash)
+        options = {column: options}
+      end
+      options ||= {}
+
+      if postgresql? && options[:algorithm] != :concurrently && !new_table?(table)
+        if StrongMigrations.safe_by_default
+          safe_remove_index(table, options)
+          throw :safe
+        end
+
+        raise_error :remove_index, command: command_str("remove_index", [table, options.merge(algorithm: :concurrently)])
+      end
+    end
+
+    def check_rename_column
+      raise_error :rename_column
+    end
+
+    def check_rename_table
+      raise_error :rename_table
+    end
+
+    def check_validate_check_constraint
+      if postgresql? && adapter.writes_blocked?
+        raise_error :validate_check_constraint
+      end
+    end
+
+    def check_validate_foreign_key
+      if postgresql? && adapter.writes_blocked?
+        raise_error :validate_foreign_key
+      end
+    end
+
+    # helpers
+
+    def postgresql?
+      adapter.instance_of?(Adapters::PostgreSQLAdapter)
+    end
+
+    def mysql?
+      adapter.instance_of?(Adapters::MySQLAdapter)
+    end
+
+    def mariadb?
+      adapter.instance_of?(Adapters::MariaDBAdapter)
+    end
+
+    def ar_version
+      ActiveRecord::VERSION::STRING.to_f
+    end
+
+    def raise_error(message_key, header: nil, append: nil, **vars)
+      return unless StrongMigrations.check_enabled?(message_key, version: version)
+
+      message = StrongMigrations.error_messages[message_key] || "Missing message"
+      message = message + append if append
+
+      vars[:migration_name] = @migration.class.name
+      vars[:migration_suffix] = "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
+      vars[:base_model] = "ApplicationRecord"
+
+      # escape % not followed by {
+      message = message.gsub(/%(?!{)/, "%%") % vars if message.include?("%")
+      @migration.stop!(message, header: header || "Dangerous operation detected")
+    end
+
+    def constraint_str(statement, identifiers)
+      # not all identifiers are tables, but this method of quoting should be fine
+      statement % identifiers.map { |v| connection.quote_table_name(v) }
+    end
+
+    def safety_assured_str(code)
+      "safety_assured do\n      execute '#{code}' \n    end"
+    end
+
+    def command_str(command, args)
+      str_args = args[0..-2].map { |a| a.inspect }
+
+      # prettier last arg
+      last_arg = args[-1]
+      if last_arg.is_a?(Hash)
+        if last_arg.any?
+          str_args << last_arg.map do |k, v|
+            if v.is_a?(Hash)
+              # pretty index: {algorithm: :concurrently}
+              "#{k}: {#{v.map { |k2, v2| "#{k2}: #{v2.inspect}" }.join(", ")}}"
+            else
+              "#{k}: #{v.inspect}"
+            end
+          end.join(", ")
+        end
+      else
+        str_args << last_arg.inspect
+      end
+
+      "#{command} #{str_args.join(", ")}"
+    end
+
+    def backfill_code(table, column, default)
+      model = table.to_s.classify
+      "#{model}.unscoped.in_batches do |relation| \n      relation.update_all #{column}: #{default.inspect}\n      sleep(0.01)\n    end"
+    end
+
+    def new_table?(table)
+      @new_tables.include?(table.to_s)
+    end
+  end
+end

data/lib/strong_migrations/error_messages.rb

@@ -0,0 +1,222 @@
+module StrongMigrations
+  self.error_messages = {
+    add_column_default:
+"Adding a column with a non-null default blocks %{rewrite_blocks} while the entire table is rewritten.
+Instead, add the column without a default value, then change the default.
+
+class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  def up
+    %{add_command}
+    %{change_command}
+  end
+
+  def down
+    %{remove_command}
+  end
+end
+
+Then backfill the existing rows in the Rails console or a separate migration with disable_ddl_transaction!.
+
+class Backfill%{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  disable_ddl_transaction!
+
+  def up
+    %{code}
+  end
+end",
+
+    add_column_default_callable:
+"Strong Migrations does not support inspecting callable default values.
+Please make really sure you're not calling a VOLATILE function,
+then wrap it in a safety_assured { ... } block.",
+
+    add_column_json:
+"There's no equality operator for the json column type, which can cause errors for
+existing SELECT DISTINCT queries in your application. Use jsonb instead.
+
+class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  def change
+    %{command}
+  end
+end",
+
+    change_column:
+"Changing the type of an existing column blocks %{rewrite_blocks}
+while the entire table is rewritten. A safer approach is to:
+
+1. Create a new column
+2. Write to both columns
+3. Backfill data from the old column to the new column
+4. Move reads from the old column to the new column
+5. Stop writing to the old column
+6. Drop the old column",
+
+    change_column_with_not_null:
+"Changing the type is safe, but setting NOT NULL is not.",
+
+    remove_column: "Active Record caches attributes, which causes problems
+when removing columns. Be sure to ignore the column%{column_suffix}:
+
+class %{model} < %{base_model}
+  %{code}
+end
+
+Deploy the code, then wrap this step in a safety_assured { ... } block.
+
+class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  def change
+    safety_assured { %{command} }
+  end
+end",
+
+    rename_column:
+"Renaming a column that's in use will cause errors
+in your application. A safer approach is to:
+
+1. Create a new column
+2. Write to both columns
+3. Backfill data from the old column to new column
+4. Move reads from the old column to the new column
+5. Stop writing to the old column
+6. Drop the old column",
+
+    rename_table:
+"Renaming a table that's in use will cause errors
+in your application. A safer approach is to:
+
+1. Create a new table. Don't forget to recreate indexes from the old table
+2. Write to both tables
+3. Backfill data from the old table to new table
+4. Move reads from the old table to the new table
+5. Stop writing to the old table
+6. Drop the old table",
+
+    add_reference:
+"%{headline} Instead, use:
+
+class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  disable_ddl_transaction!
+
+  def change
+    %{command}
+  end
+end",
+
+    add_index:
+"Adding an index non-concurrently blocks writes. Instead, use:
+
+class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  disable_ddl_transaction!
+
+  def change
+    %{command}
+  end
+end",
+
+    remove_index:
+"Removing an index non-concurrently blocks writes. Instead, use:
+
+class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  disable_ddl_transaction!
+
+  def change
+    %{command}
+  end
+end",
+
+    add_index_columns:
+"Adding a non-unique index with more than three columns rarely improves performance.
+Instead, start an index with columns that narrow down the results the most.",
+
+    change_table:
+"Strong Migrations does not support inspecting what happens inside a
+change_table block, so cannot help you here. Please make really sure that what
+you're doing is safe before proceeding, then wrap it in a safety_assured { ... } block.",
+
+    create_table:
+"The force option will destroy existing tables.
+If this is intended, drop the existing table first.
+Otherwise, remove the force option.",
+
+    execute:
+"Strong Migrations does not support inspecting what happens inside an
+execute call, so cannot help you here. Please make really sure that what
+you're doing is safe before proceeding, then wrap it in a safety_assured { ... } block.",
+
+    change_column_null:
+"Passing a default value to change_column_null runs a single UPDATE query,
+which can cause downtime. Instead, backfill the existing rows in the
+Rails console or a separate migration with disable_ddl_transaction!.
+
+class Backfill%{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  disable_ddl_transaction!
+
+  def up
+    %{code}
+  end
+end",
+
+    change_column_null_postgresql:
+"Setting NOT NULL on an existing column blocks reads and writes while every row is checked.
+Instead, add a check constraint and validate it in a separate migration.
+
+class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  def change
+    %{add_constraint_code}
+  end
+end
+
+class Validate%{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  %{validate_constraint_code}
+end",
+
+    change_column_null_mysql:
+"Setting NOT NULL on an existing column is not safe without strict mode enabled.",
+
+    add_foreign_key:
+"Adding a foreign key blocks writes on both tables. Instead,
+add the foreign key without validating existing rows,
+then validate them in a separate migration.
+
+class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  def change
+    %{add_foreign_key_code}
+  end
+end
+
+class Validate%{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  def change
+    %{validate_foreign_key_code}
+  end
+end",
+
+    validate_foreign_key:
+"Validating a foreign key while writes are blocked is dangerous.
+Use disable_ddl_transaction! or a separate migration.",
+
+    add_check_constraint:
+"Adding a check constraint key blocks reads and writes while every row is checked.
+Instead, add the check constraint without validating existing rows,
+then validate them in a separate migration.
+
+class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  def change
+    %{add_check_constraint_code}
+  end
+end
+
+class Validate%{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  def change
+    %{validate_check_constraint_code}
+  end
+end",
+
+    add_check_constraint_mysql:
+"Adding a check constraint to an existing table is not safe with your database engine.",
+
+    validate_check_constraint:
+"Validating a check constraint while writes are blocked is dangerous.
+Use disable_ddl_transaction! or a separate migration."
+  }
+  self.enabled_checks = (error_messages.keys - [:remove_index]).map { |k| [k, {}] }.to_h
+end

data/lib/strong_migrations/migration.rb

@@ -12,8 +12,10 @@ module StrongMigrations
       # Active Record 7.0.2+ versioned schema
       return super if defined?(ActiveRecord::Schema::Definition) && is_a?(ActiveRecord::Schema::Definition)
 
-      strong_migrations_checker.perform(method, *args) do
-        super
+      catch(:safe) do
+        strong_migrations_checker.perform(method, *args) do
+          super
+        end
       end
     end
     ruby2_keywords(:method_missing) if respond_to?(:ruby2_keywords, true)

data/lib/strong_migrations/migrator.rb

@@ -0,0 +1,19 @@
+module StrongMigrations
+  module Migrator
+    def ddl_transaction(migration, *args)
+      return super unless StrongMigrations.lock_timeout_retries > 0 && use_transaction?(migration)
+
+      # handle MigrationProxy class
+      migration = migration.send(:migration) if migration.respond_to?(:migration, true)
+
+      # retry migration since the entire transaction needs to be rerun
+      checker = migration.send(:strong_migrations_checker)
+      checker.retry_lock_timeouts(check_committed: true) do
+        # failed transaction reverts timeout, so need to re-apply
+        checker.timeouts_set = false
+
+        super(migration, *args)
+      end
+    end
+  end
+end

data/lib/strong_migrations/safe_methods.rb

@@ -69,9 +69,13 @@ module StrongMigrations
       end
     end
 
-    def safe_change_column_null(add_code, validate_code, change_args, remove_code)
+    def safe_change_column_null(add_code, validate_code, change_args, remove_code, default)
       @migration.reversible do |dir|
         dir.up do
+          unless default.nil?
+            raise Error, "default value not supported yet with safe_by_default"
+          end
+
           @migration.safety_assured do
             @migration.execute(add_code)
             disable_transaction

data/lib/strong_migrations/version.rb

@@ -1,3 +1,3 @@
 module StrongMigrations
-  VERSION = "0.8.0"
+  VERSION = "1.0.0"
 end