strong_migrations 0.6.5 → 0.7.1

data/lib/generators/strong_migrations/install_generator.rb

@@ -0,0 +1,28 @@
+require "rails/generators"
+
+module StrongMigrations
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      source_root File.join(__dir__, "templates")
+
+      def create_initializer
+        template "initializer.rb", "config/initializers/strong_migrations.rb"
+      end
+
+      def start_after
+        Time.now.utc.strftime("%Y%m%d%H%M%S")
+      end
+
+      def target_version
+        case ActiveRecord::Base.connection_config[:adapter].to_s
+        when /mysql/
+          # could try to connect to database and check for MariaDB
+          # but this should be fine
+          '"8.0.12"'
+        else
+          "10"
+        end
+      end
+    end
+  end
+end

data/lib/generators/strong_migrations/templates/initializer.rb.tt

@@ -0,0 +1,22 @@
+# Mark existing migrations as safe
+StrongMigrations.start_after = <%= start_after %>
+
+# Set timeouts for migrations
+# If you use PgBouncer in transaction mode, delete these lines and set timeouts on the database user
+StrongMigrations.lock_timeout = 10.seconds
+StrongMigrations.statement_timeout = 1.hour
+
+# Analyze tables after indexes are added
+# Outdated statistics can sometimes hurt performance
+StrongMigrations.auto_analyze = true
+
+# Set the version of the production database
+# so the right checks are run in development
+# StrongMigrations.target_version = <%= target_version %>
+
+# Add custom checks
+# StrongMigrations.add_check do |method, args|
+#   if method == :add_index && args[0].to_s == "users"
+#     stop! "No more indexes on the users table"
+#   end
+# end

data/lib/strong_migrations.rb

@@ -5,7 +5,6 @@ require "active_support"
 require "strong_migrations/checker"
 require "strong_migrations/database_tasks"
 require "strong_migrations/migration"
-require "strong_migrations/migration_helpers"
 require "strong_migrations/version"
 
 # integrations
@@ -18,14 +17,15 @@ module StrongMigrations
   class << self
     attr_accessor :auto_analyze, :start_after, :checks, :error_messages,
       :target_postgresql_version, :target_mysql_version, :target_mariadb_version,
-      :enabled_checks, :lock_timeout, :statement_timeout, :helpers
+      :enabled_checks, :lock_timeout, :statement_timeout, :check_down, :target_version
+    attr_writer :lock_timeout_limit
   end
   self.auto_analyze = false
   self.start_after = 0
   self.checks = []
   self.error_messages = {
     add_column_default:
-"Adding a column with a non-null default causes the entire table to be rewritten.
+"Adding a column with a non-null default blocks %{rewrite_blocks} while the entire table is rewritten.
 Instead, add the column without a default value, then change the default.
 
 class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
@@ -50,12 +50,18 @@ class Backfill%{migration_name} < ActiveRecord::Migration%{migration_suffix}
 end",
 
     add_column_json:
-"There's no equality operator for the json column type, which can
-cause errors for existing SELECT DISTINCT queries. Use jsonb instead.",
+"There's no equality operator for the json column type, which can cause errors for
+existing SELECT DISTINCT queries in your application. Use jsonb instead.
+
+class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  def change
+    %{command}
+  end
+end",
 
     change_column:
-"Changing the type of an existing column requires the entire
-table and indexes to be rewritten. A safer approach is to:
+"Changing the type of an existing column blocks %{rewrite_blocks}
+while the entire table is rewritten. A safer approach is to:
 
 1. Create a new column
 2. Write to both columns
@@ -64,7 +70,10 @@ table and indexes to be rewritten. A safer approach is to:
 5. Stop writing to the old column
 6. Drop the old column",
 
-    remove_column: "Active Record caches attributes which causes problems
+    change_column_with_not_null:
+"Changing the type is safe, but setting NOT NULL is not.",
+
+    remove_column: "Active Record caches attributes, which causes problems
 when removing columns. Be sure to ignore the column%{column_suffix}:
 
 class %{model} < %{base_model}
@@ -80,7 +89,8 @@ class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
 end",
 
     rename_column:
-"Renaming a column is dangerous. A safer approach is to:
+"Renaming a column that's in use will cause errors
+in your application. A safer approach is to:
 
 1. Create a new column
 2. Write to both columns
@@ -90,7 +100,8 @@ end",
 6. Drop the old column",
 
     rename_table:
-"Renaming a table is dangerous. A safer approach is to:
+"Renaming a table that's in use will cause errors
+in your application. A safer approach is to:
 
 1. Create a new table. Don't forget to recreate indexes from the old table
 2. Write to both tables
@@ -111,7 +122,7 @@ class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
 end",
 
     add_index:
-"Adding an index non-concurrently locks the table. Instead, use:
+"Adding an index non-concurrently blocks writes. Instead, use:
 
 class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
   disable_ddl_transaction!
@@ -122,7 +133,7 @@ class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
 end",
 
     remove_index:
-"Removing an index non-concurrently locks the table. Instead, use:
+"Removing an index non-concurrently blocks writes. Instead, use:
 
 class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
   disable_ddl_transaction!
@@ -165,9 +176,8 @@ class Backfill%{migration_name} < ActiveRecord::Migration%{migration_suffix}
 end",
 
     change_column_null_postgresql:
-"Setting NOT NULL on a column requires an AccessExclusiveLock,
-which is expensive on large tables. Instead, use a constraint and
-validate it in a separate migration with a more agreeable RowShareLock.
+"Setting NOT NULL on an existing column blocks reads and writes while every row is checked.
+Instead, add a check constraint and validate it in a separate migration.
 
 class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
   def change
@@ -181,26 +191,13 @@ class Validate%{migration_name} < ActiveRecord::Migration%{migration_suffix}
   end
 end",
 
-    change_column_null_postgresql_helper:
-"Setting NOT NULL on a column requires an AccessExclusiveLock,
-which is expensive on large tables. Instead, we can use a constraint and
-validate it in a separate step with a more agreeable RowShareLock.
-
-class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
-  disable_ddl_transaction!
-
-  def change
-    %{command}
-  end
-end",
-
     change_column_null_mysql:
 "Setting NOT NULL on an existing column is not safe with your database engine.",
 
     add_foreign_key:
-"New foreign keys are validated by default. This acquires an AccessExclusiveLock,
-which is expensive on large tables. Instead, validate it in a separate migration
-with a more agreeable RowShareLock.
+"Adding a foreign key blocks writes on both tables. Instead,
+add the foreign key without validating existing rows,
+then validate them in a separate migration.
 
 class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
   def change
@@ -214,21 +211,24 @@ class Validate%{migration_name} < ActiveRecord::Migration%{migration_suffix}
   end
 end",
 
-    add_foreign_key_helper:
-"New foreign keys are validated by default. This acquires an AccessExclusiveLock,
-which is expensive on large tables. Instead, we can validate it in a separate step
-with a more agreeable RowShareLock.
+    validate_foreign_key:
+"Validating a foreign key while writes are blocked is dangerous.
+Use disable_ddl_transaction! or a separate migration."
+  }
+  self.enabled_checks = (error_messages.keys - [:remove_index]).map { |k| [k, {}] }.to_h
+  self.check_down = false
 
-class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
-  disable_ddl_transaction!
+  # private
+  def self.developer_env?
+    defined?(Rails) && (Rails.env.development? || Rails.env.test?)
+  end
 
-  def change
-    %{command}
+  def self.lock_timeout_limit
+    unless defined?(@lock_timeout_limit)
+      @lock_timeout_limit = developer_env? ? false : 10
+    end
+    @lock_timeout_limit
   end
-end",
-  }
-  self.enabled_checks = (error_messages.keys - [:remove_index]).map { |k| [k, {}] }.to_h
-  self.helpers = false
 
   def self.add_check(&block)
     checks << block
@@ -250,13 +250,6 @@ end",
       false
     end
   end
-
-  # def self.enable_helpers
-  #   unless helpers
-  #     ActiveRecord::Migration.include(StrongMigrations::MigrationHelpers)
-  #     self.helpers = true
-  #   end
-  # end
 end
 
 ActiveSupport.on_load(:active_record) do

data/lib/strong_migrations/checker.rb

@@ -7,6 +7,7 @@ module StrongMigrations
       @new_tables = []
       @safe = false
       @timeouts_set = false
+      @lock_timeout_checked = false
     end
 
     def safety_assured
@@ -21,6 +22,7 @@ module StrongMigrations
 
     def perform(method, *args)
       set_timeouts
+      check_lock_timeout
 
       unless safe?
         case method
@@ -94,11 +96,13 @@ Then add the NOT NULL constraint in separate migrations."
               change_command: command_str("change_column_default", [table, column, default]),
               remove_command: command_str("remove_column", [table, column]),
               code: backfill_code(table, column, default),
-              append: append
+              append: append,
+              rewrite_blocks: rewrite_blocks
           end
 
           if type.to_s == "json" && postgresql?
-            raise_error :add_column_json
+            raise_error :add_column_json,
+              command: command_str("add_column", [table, column, :jsonb, options])
           end
         when :change_column
           table, column, type, options = args
@@ -107,15 +111,24 @@ Then add the NOT NULL constraint in separate migrations."
           safe = false
           existing_column = connection.columns(table).find { |c| c.name.to_s == column.to_s }
           if existing_column
-            sql_type = existing_column.sql_type.split("(").first
+            existing_type = existing_column.sql_type.split("(").first
             if postgresql?
               case type.to_s
-              when "string", "text"
-                # safe to change limit for varchar
-                safe = ["character varying", "text"].include?(sql_type)
+              when "string"
+                # safe to increase limit or remove it
+                # not safe to decrease limit or add a limit
+                case existing_type
+                when "character varying"
+                  safe = !options[:limit] || (existing_column.limit && options[:limit] >= existing_column.limit)
+                when "text"
+                  safe = !options[:limit]
+                end
+              when "text"
+                # safe to change varchar to text (and text to text)
+                safe = ["character varying", "text"].include?(existing_type)
               when "numeric", "decimal"
                 # numeric and decimal are equivalent and can be used interchangably
-                safe = ["numeric", "decimal"].include?(sql_type) &&
+                safe = ["numeric", "decimal"].include?(existing_type) &&
                   (
                     (
                       # unconstrained
@@ -128,7 +141,7 @@ Then add the NOT NULL constraint in separate migrations."
                     )
                   )
               when "datetime", "timestamp", "timestamptz"
-                safe = ["timestamp without time zone", "timestamp with time zone"].include?(sql_type) &&
+                safe = ["timestamp without time zone", "timestamp with time zone"].include?(existing_type) &&
                   postgresql_version >= Gem::Version.new("12") &&
                   connection.select_all("SHOW timezone").first["TimeZone"] == "UTC"
               end
@@ -140,13 +153,19 @@ Then add the NOT NULL constraint in separate migrations."
                 # increased limit, but doesn't change number of length bytes
                 # 1-255 = 1 byte, 256-65532 = 2 bytes, 65533+ = too big for varchar
                 limit = options[:limit] || 255
-                safe = ["varchar"].include?(sql_type) &&
+                safe = ["varchar"].include?(existing_type) &&
                   limit >= existing_column.limit &&
                   (limit <= 255 || existing_column.limit > 255)
               end
             end
           end
-          raise_error :change_column unless safe
+
+          # unsafe to set NOT NULL for safe types
+          if safe && existing_column.null && options[:null] == false
+            raise_error :change_column_with_not_null
+          end
+
+          raise_error :change_column, rewrite_blocks: rewrite_blocks unless safe
         when :create_table
           table, options = args
           options ||= {}
@@ -174,7 +193,7 @@ Then add the NOT NULL constraint in separate migrations."
               end
 
               if options.delete(:foreign_key)
-                headline = "Adding a validated foreign key locks the table."
+                headline = "Adding a foreign key blocks writes on both tables."
                 append = "
 
 Then add the foreign key in separate migrations."
@@ -194,16 +213,25 @@ Then add the foreign key in separate migrations."
           table, column, null, default = args
           if !null
             if postgresql?
-              if helpers?
-                raise_error :change_column_null_postgresql_helper,
-                  command: command_str(:add_null_constraint_safely, [table, column])
-              else
+              safe = false
+              if postgresql_version >= Gem::Version.new("12")
+                # TODO likely need to quote the column in some situations
+                safe = constraints(table).any? { |c| c["def"] == "CHECK ((#{column} IS NOT NULL))" }
+              end
+
+              unless safe
                 # match https://github.com/nullobject/rein
                 constraint_name = "#{table}_#{column}_null"
 
+                validate_constraint_code = String.new(constraint_str("ALTER TABLE %s VALIDATE CONSTRAINT %s", [table, constraint_name]))
+                if postgresql_version >= Gem::Version.new("12")
+                  validate_constraint_code << "\n    #{command_str(:change_column_null, [table, column, null])}"
+                  validate_constraint_code << "\n    #{constraint_str("ALTER TABLE %s DROP CONSTRAINT %s", [table, constraint_name])}"
+                end
+
                 raise_error :change_column_null_postgresql,
                   add_constraint_code: constraint_str("ALTER TABLE %s ADD CONSTRAINT %s CHECK (%s IS NOT NULL) NOT VALID", [table, constraint_name, column]),
-                  validate_constraint_code: constraint_str("ALTER TABLE %s VALIDATE CONSTRAINT %s", [table, constraint_name])
+                  validate_constraint_code: validate_constraint_code
               end
             elsif mysql? || mariadb?
               raise_error :change_column_null_mysql
@@ -220,10 +248,7 @@ Then add the foreign key in separate migrations."
           validate = options.fetch(:validate, true) || ActiveRecord::VERSION::STRING < "5.2"
 
           if postgresql? && validate
-            if helpers?
-              raise_error :add_foreign_key_helper,
-                command: command_str(:add_foreign_key_safely, [from_table, to_table, options])
-            elsif ActiveRecord::VERSION::STRING < "5.2"
+            if ActiveRecord::VERSION::STRING < "5.2"
               # fk name logic from rails
               primary_key = options[:primary_key] || "id"
               column = options[:column] || "#{to_table.to_s.singularize}_id"
@@ -239,6 +264,10 @@ Then add the foreign key in separate migrations."
                 validate_foreign_key_code: command_str("validate_foreign_key", [from_table, to_table])
             end
           end
+        when :validate_foreign_key
+          if postgresql? && writes_blocked?
+            raise_error :validate_foreign_key
+          end
         end
 
         StrongMigrations.checks.each do |check|
@@ -248,9 +277,10 @@ Then add the foreign key in separate migrations."
 
       result = yield
 
+      # outdated statistics + a new index can hurt performance of existing queries
       if StrongMigrations.auto_analyze && direction == :up && method == :add_index
         if postgresql?
-          connection.execute "ANALYZE VERBOSE #{connection.quote_table_name(args[0].to_s)}"
+          connection.execute "ANALYZE #{connection.quote_table_name(args[0].to_s)}"
         elsif mariadb? || mysql?
           connection.execute "ANALYZE TABLE #{connection.quote_table_name(args[0].to_s)}"
         end
@@ -259,12 +289,14 @@ Then add the foreign key in separate migrations."
       result
     end
 
+    private
+
     def set_timeouts
       if !@timeouts_set
         if StrongMigrations.statement_timeout
           statement =
             if postgresql?
-              "SET statement_timeout TO #{connection.quote(StrongMigrations.statement_timeout.to_i * 1000)}"
+              "SET statement_timeout TO #{connection.quote(postgresql_timeout(StrongMigrations.statement_timeout))}"
             elsif mysql?
               "SET max_execution_time = #{connection.quote(StrongMigrations.statement_timeout.to_i * 1000)}"
             elsif mariadb?
@@ -279,7 +311,7 @@ Then add the foreign key in separate migrations."
         if StrongMigrations.lock_timeout
           statement =
             if postgresql?
-              "SET lock_timeout TO #{connection.quote(StrongMigrations.lock_timeout.to_i * 1000)}"
+              "SET lock_timeout TO #{connection.quote(postgresql_timeout(StrongMigrations.lock_timeout))}"
             elsif mysql? || mariadb?
               "SET lock_wait_timeout = #{connection.quote(StrongMigrations.lock_timeout)}"
             else
@@ -293,8 +325,6 @@ Then add the foreign key in separate migrations."
       end
     end
 
-    private
-
     def connection
       @migration.connection
     end
@@ -304,7 +334,8 @@ Then add the foreign key in separate migrations."
     end
 
     def safe?
-      @safe || ENV["SAFETY_ASSURED"] || @migration.is_a?(ActiveRecord::Schema) || direction == :down || version_safe?
+      @safe || ENV["SAFETY_ASSURED"] || @migration.is_a?(ActiveRecord::Schema) ||
+        (direction == :down && !StrongMigrations.check_down) || version_safe?
     end
 
     def version_safe?
@@ -349,8 +380,9 @@ Then add the foreign key in separate migrations."
     end
 
     def target_version(target_version)
+      target_version ||= StrongMigrations.target_version
       version =
-        if target_version && defined?(Rails) && (Rails.env.development? || Rails.env.test?)
+        if target_version && StrongMigrations.developer_env?
           target_version.to_s
         else
           yield
@@ -358,8 +390,69 @@ Then add the foreign key in separate migrations."
       Gem::Version.new(version)
     end
 
-    def helpers?
-      StrongMigrations.helpers
+    def check_lock_timeout
+      limit = StrongMigrations.lock_timeout_limit
+
+      if limit && !@lock_timeout_checked
+        if postgresql?
+          lock_timeout = connection.select_all("SHOW lock_timeout").first["lock_timeout"]
+          lock_timeout_sec = timeout_to_sec(lock_timeout)
+          if lock_timeout_sec == 0
+            warn "[strong_migrations] DANGER: No lock timeout set"
+          elsif lock_timeout_sec > limit
+            warn "[strong_migrations] DANGER: Lock timeout is longer than #{limit} seconds: #{lock_timeout}"
+          end
+        elsif mysql? || mariadb?
+          lock_timeout = connection.select_all("SHOW VARIABLES LIKE 'lock_wait_timeout'").first["Value"]
+          if lock_timeout.to_i > limit
+            warn "[strong_migrations] DANGER: Lock timeout is longer than #{limit} seconds: #{lock_timeout}"
+          end
+        end
+        @lock_timeout_checked = true
+      end
+    end
+
+    # units: https://www.postgresql.org/docs/current/config-setting.html
+    def timeout_to_sec(timeout)
+      units = {
+        "us" => 0.001,
+        "ms" => 1,
+        "s" => 1000,
+        "min" => 1000 * 60,
+        "h" => 1000 * 60 * 60,
+        "d" => 1000 * 60 * 60 * 24
+      }
+      timeout_ms = timeout.to_i
+      units.each do |k, v|
+        if timeout.end_with?(k)
+          timeout_ms *= v
+          break
+        end
+      end
+      timeout_ms / 1000.0
+    end
+
+    def postgresql_timeout(timeout)
+      if timeout.is_a?(String)
+        timeout
+      else
+        timeout.to_i * 1000
+      end
+    end
+
+    def constraints(table_name)
+      query = <<~SQL
+        SELECT
+          conname AS name,
+          pg_get_constraintdef(oid) AS def
+        FROM
+          pg_constraint
+        WHERE
+          contype = 'c' AND
+          convalidated AND
+          conrelid = #{connection.quote(connection.quote_table_name(table_name))}::regclass
+      SQL
+      connection.select_all(query.squish).to_a
     end
 
     def raise_error(message_key, header: nil, append: nil, **vars)
@@ -406,6 +499,23 @@ Then add the foreign key in separate migrations."
       "#{command} #{str_args.join(", ")}"
     end
 
+    def writes_blocked?
+      query = <<~SQL
+        SELECT
+          relation::regclass::text
+        FROM
+          pg_locks
+        WHERE
+          mode IN ('ShareRowExclusiveLock', 'AccessExclusiveLock') AND
+          pid = pg_backend_pid()
+      SQL
+      connection.select_all(query.squish).any?
+    end
+
+    def rewrite_blocks
+      mysql? || mariadb? ? "writes" : "reads and writes"
+    end
+
     def backfill_code(table, column, default)
       model = table.to_s.classify
       "#{model}.unscoped.in_batches do |relation| \n      relation.update_all #{column}: #{default.inspect}\n      sleep(0.01)\n    end"