online_migrations 0.9.2 → 0.11.0

data/lib/online_migrations/command_checker.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "erb"
-require "openssl"
 require "set"
 
 module OnlineMigrations
@@ -53,6 +52,36 @@ module OnlineMigrations
     end
 
     private
+      ERROR_MESSAGE_TO_LINK = {
+        multiple_foreign_keys: "adding-multiple-foreign-keys",
+        create_table: "creating-a-table-with-the-force-option",
+        short_primary_key_type: "using-primary-key-with-short-integer-type",
+        drop_table_multiple_foreign_keys: "removing-a-table-with-multiple-foreign-keys",
+        rename_table: "renaming-a-table",
+        add_column_with_default_null: "adding-a-column-with-a-default-value",
+        add_column_with_default: "adding-a-column-with-a-default-value",
+        add_column_generated_stored: "adding-a-stored-generated-column",
+        add_column_json: "adding-a-json-column",
+        rename_column: "renaming-a-column",
+        change_column: "changing-the-type-of-a-column",
+        change_column_default: "changing-the-default-value-of-a-column",
+        change_column_null: "setting-not-null-on-an-existing-column",
+        remove_column: "removing-a-column",
+        add_timestamps_with_default: "adding-a-column-with-a-default-value",
+        add_hash_index: "hash-indexes",
+        add_reference: "adding-a-reference",
+        add_index: "adding-an-index-non-concurrently",
+        replace_index: "replacing-an-index",
+        remove_index: "removing-an-index-non-concurrently",
+        add_foreign_key: "adding-a-foreign-key",
+        add_exclusion_constraint: "adding-an-exclusion-constraint",
+        add_check_constraint: "adding-a-check-constraint",
+        add_unique_constraint: "adding-a-unique-constraint",
+        execute: "executing-SQL-directly",
+        add_inheritance_column: "adding-a-single-table-inheritance-column",
+        mismatched_foreign_key_type: "mismatched-reference-column-types",
+      }
+
       def check_database_version
         return if defined?(@database_version_checked)
 
@@ -125,13 +154,7 @@ module OnlineMigrations
           check_columns_removal(command, *args, **options)
         else
           if respond_to?(command, true)
-            if options.any?
-              # Workaround for Active Record < 5 change_column_default
-              # not accepting options.
-              send(command, *args, **options, &block)
-            else
-              send(command, *args, &block)
-            end
+            send(command, *args, **options, &block)
           else
             # assume it is safe
             true
@@ -333,9 +356,7 @@ module OnlineMigrations
               precision = options[:precision] || options[:limit] || 6
               existing_precision = existing_column.precision || existing_column.limit || 6
 
-              # PostgreSQL interval data type was added in https://github.com/rails/rails/pull/16919
-              (existing_type == :interval || (Utils.ar_version < 6.1 && existing_column.sql_type.start_with?("interval"))) &&
-              precision >= existing_precision
+              existing_type == :interval && precision >= existing_precision
             when :inet
               existing_type == :cidr
             else
@@ -458,7 +479,7 @@ module OnlineMigrations
 
       def add_reference(table_name, ref_name, **options)
         # Always added by default in 5.0+
-        index = options.fetch(:index) { Utils.ar_version >= 5.0 }
+        index = options.fetch(:index, true)
 
         if index.is_a?(Hash) && index[:using].to_s == "hash" && postgresql_version < Gem::Version.new("10")
           raise_error :add_hash_index
@@ -486,7 +507,7 @@ module OnlineMigrations
         end
 
         if !options[:polymorphic]
-          type = (options[:type] || (Utils.ar_version >= 5.1 ? :bigint : :integer)).to_sym
+          type = (options[:type] || :bigint).to_sym
           column_name = "#{ref_name}_id"
 
           foreign_key_options = foreign_key.is_a?(Hash) ? foreign_key : {}
@@ -522,6 +543,11 @@ module OnlineMigrations
               end
             end
           end
+
+          # Outdated statistics + a new index can hurt performance of existing queries.
+          if OnlineMigrations.config.auto_analyze && direction == :up
+            connection.execute("ANALYZE #{table_name}")
+          end
         end
       end
 
@@ -539,9 +565,9 @@ module OnlineMigrations
         end
 
         if index_def
-          existing_options = [:name, :columns, :unique, :where, :type, :using, :opclasses].map do |option|
+          existing_options = [:name, :columns, :unique, :where, :type, :using, :opclasses].filter_map do |option|
             [option, index_def.public_send(option)] if index_def.respond_to?(option)
-          end.compact.to_h
+          end.to_h
 
           @removed_indexes << IndexDefinition.new(table: table_name, **existing_options)
         end
@@ -586,7 +612,7 @@ module OnlineMigrations
       def add_unique_constraint(table_name, column_name = nil, **options)
         return if new_or_small_table?(table_name) || options[:using_index] || !column_name
 
-        index_name = index_name(table_name, column_name)
+        index_name = Utils.index_name(table_name, column_name)
 
         raise_error :add_unique_constraint,
           add_index_code: command_str(:add_index, table_name, column_name, unique: true, name: index_name, algorithm: :concurrently),
@@ -594,22 +620,6 @@ module OnlineMigrations
           remove_code: command_str(:remove_unique_constraint, table_name, column_name)
       end
 
-      # Implementation is from Active Record
-      def index_name(table_name, column_name)
-        max_index_name_size = 62
-        name = "index_#{table_name}_on_#{Array(column_name) * '_and_'}"
-        return name if name.bytesize <= max_index_name_size
-
-        # Fallback to short version, add hash to ensure uniqueness
-        hashed_identifier = "_#{OpenSSL::Digest::SHA256.hexdigest(name).first(10)}"
-        name = "idx_on_#{Array(column_name) * '_'}"
-
-        short_limit = max_index_name_size - hashed_identifier.bytesize
-        short_name = name[0, short_limit]
-
-        "#{short_name}#{hashed_identifier}"
-      end
-
       def validate_constraint(*)
         if crud_blocked?
           raise_error :validate_constraint
@@ -720,19 +730,13 @@ module OnlineMigrations
         template = OnlineMigrations.config.error_messages.fetch(message_key)
 
         vars[:migration_name] = @migration.name
-        vars[:migration_parent] = Utils.migration_parent_string
-        vars[:model_parent] = Utils.model_parent_string
+        vars[:migration_parent] = "ActiveRecord::Migration[#{Utils.ar_version}]"
         vars[:ar_version] = Utils.ar_version
 
-        if RUBY_VERSION >= "2.6"
-          message = ERB.new(template, trim_mode: "<>").result_with_hash(vars)
-        else
-          # `result_with_hash` was added in ruby 2.5
-          b = TOPLEVEL_BINDING.dup
-          vars.each_pair do |key, value|
-            b.local_variable_set(key, value)
-          end
-          message = ERB.new(template, nil, "<>").result(b)
+        message = ERB.new(template, trim_mode: "<>").result_with_hash(vars)
+
+        if (link = ERROR_MESSAGE_TO_LINK[message_key])
+          message += "\nFor more details, see https://github.com/fatkodima/online_migrations##{link}"
         end
 
         @migration.stop!(message, header: header || "Dangerous operation detected")
@@ -768,7 +772,7 @@ module OnlineMigrations
       end
 
       def crud_blocked?
-        locks_query = <<-SQL.strip_heredoc
+        locks_query = <<~SQL
           SELECT relation::regclass::text
           FROM pg_locks
           WHERE mode IN ('ShareLock', 'ShareRowExclusiveLock', 'ExclusiveLock', 'AccessExclusiveLock')
@@ -786,7 +790,7 @@ module OnlineMigrations
       end
 
       def check_constraints(table_name)
-        constraints_query = <<-SQL.strip_heredoc
+        constraints_query = <<~SQL
           SELECT pg_get_constraintdef(oid) AS def
           FROM pg_constraint
           WHERE contype = 'c'
@@ -807,7 +811,7 @@ module OnlineMigrations
 
       def check_mismatched_foreign_key_type(table_name, column_name, type, **options)
         column_name = column_name.to_s
-        ref_name = column_name.sub(/_id\z/, "")
+        ref_name = column_name.delete_suffix("_id")
 
         if like_foreign_key?(column_name, type)
           foreign_table_name = Utils.foreign_table_name(ref_name, options)

data/lib/online_migrations/config.rb

@@ -16,7 +16,6 @@ module OnlineMigrations
     #
     def start_after=(value)
       if value.is_a?(Hash)
-        ensure_supports_multiple_dbs
         @start_after = value.stringify_keys
       else
         @start_after = value
@@ -49,7 +48,6 @@ module OnlineMigrations
     #
     def target_version=(value)
       if value.is_a?(Hash)
-        ensure_supports_multiple_dbs
         @target_version = value.stringify_keys
       else
         @target_version = value
@@ -84,6 +82,16 @@ module OnlineMigrations
     #
     attr_accessor :error_messages
 
+    # Whether to automatically run ANALYZE on the table after the index was added
+    # @return [Boolean]
+    #
+    attr_accessor :auto_analyze
+
+    # Whether to alphabetize schema
+    # @return [Boolean]
+    #
+    attr_accessor :alphabetize_schema
+
     # Maximum allowed lock timeout value (in seconds)
     #
     # If set lock timeout is greater than this value, the migration will fail.
@@ -138,7 +146,7 @@ module OnlineMigrations
     # Returns a list of enabled checks
     #
     # All checks are enabled by default. To disable/enable a check use `disable_check`/`enable_check`.
-    # For the list of available checks look at `lib/error_messages` folder.
+    # For the list of available checks look at the `error_messages.rb` file.
     #
     # @return [Array]
     #
@@ -174,7 +182,7 @@ module OnlineMigrations
         attempts: 30,
         base_delay: 0.01.seconds,
         max_delay: 1.minute,
-        lock_timeout: 0.05.seconds
+        lock_timeout: 0.2.seconds
       )
 
       @background_migrations = BackgroundMigrations::Config.new
@@ -184,8 +192,10 @@ module OnlineMigrations
       @target_version = nil
       @small_tables = []
       @check_down = false
-      @enabled_checks = @error_messages.keys.map { |k| [k, {}] }.to_h
-      @verbose_sql_logs = defined?(Rails.env) && Rails.env.production?
+      @auto_analyze = false
+      @alphabetize_schema = false
+      @enabled_checks = @error_messages.keys.index_with({})
+      @verbose_sql_logs = defined?(Rails.env) && (Rails.env.production? || Rails.env.staging?)
     end
 
     def lock_retrier=(value)
@@ -198,7 +208,7 @@ module OnlineMigrations
 
     # Enables specific check
     #
-    # For the list of available checks look at `lib/error_messages` module.
+    # For the list of available checks look at the `error_messages.rb` file.
     #
     # @param name [Symbol] check name
     # @param start_after [Integer] migration version from which this check will be performed
@@ -210,7 +220,7 @@ module OnlineMigrations
 
     # Disables specific check
     #
-    # For the list of available checks look at `lib/error_messages` module.
+    # For the list of available checks look at the `error_messages.rb` file.
     #
     # @param name [Symbol] check name
     # @return [void]
@@ -221,7 +231,7 @@ module OnlineMigrations
 
     # Test whether specific check is enabled
     #
-    # For the list of available checks look at `lib/error_messages` module.
+    # For the list of available checks look at the `error_messages.rb` file.
     #
     # @param name [Symbol] check name
     # @param version [Integer] migration version
@@ -260,12 +270,6 @@ module OnlineMigrations
     end
 
     private
-      def ensure_supports_multiple_dbs
-        if !Utils.supports_multiple_dbs?
-          raise "OnlineMigrations does not support multiple databases for Active Record < 6.1"
-        end
-      end
-
       def db_config_name
         connection = OnlineMigrations.current_migration.connection
         connection.pool.db_config.name

data/lib/online_migrations/copy_trigger.rb

@@ -18,12 +18,12 @@ module OnlineMigrations
       "trigger_#{hashed_identifier}"
     end
 
-    def create(from_columns, to_columns)
+    def create(from_columns, to_columns, type_cast_functions: {})
       from_columns, to_columns = normalize_column_names(from_columns, to_columns)
       trigger_name = name(from_columns, to_columns)
-      assignment_clauses = assignment_clauses_for_columns(from_columns, to_columns)
+      assignment_clauses = assignment_clauses_for_columns(from_columns, to_columns, type_cast_functions)
 
-      connection.execute(<<-SQL.strip_heredoc)
+      connection.execute(<<~SQL)
         CREATE OR REPLACE FUNCTION #{trigger_name}() RETURNS TRIGGER AS $$
         BEGIN
           #{assignment_clauses};
@@ -32,11 +32,11 @@ module OnlineMigrations
         $$ LANGUAGE plpgsql;
       SQL
 
-      connection.execute(<<-SQL.strip_heredoc)
+      connection.execute(<<~SQL)
         DROP TRIGGER IF EXISTS #{trigger_name} ON #{quoted_table_name}
       SQL
 
-      connection.execute(<<-SQL.strip_heredoc)
+      connection.execute(<<~SQL)
         CREATE TRIGGER #{trigger_name}
           BEFORE INSERT OR UPDATE
           ON #{quoted_table_name}
@@ -75,14 +75,19 @@ module OnlineMigrations
         [from_columns, to_columns]
       end
 
-      def assignment_clauses_for_columns(from_columns, to_columns)
+      def assignment_clauses_for_columns(from_columns, to_columns, type_cast_functions)
         combined_column_names = to_columns.zip(from_columns)
 
         assignment_clauses = combined_column_names.map do |(new_name, old_name)|
-          new_name = connection.quote_column_name(new_name)
-          old_name = connection.quote_column_name(old_name)
-
-          "NEW.#{new_name} := NEW.#{old_name}"
+          quoted_new_name = connection.quote_column_name(new_name)
+          quoted_old_name = connection.quote_column_name(old_name)
+          type_cast_function = type_cast_functions[old_name]
+
+          if type_cast_function
+            "NEW.#{quoted_new_name} := #{type_cast_function.gsub(old_name.to_s, "NEW.#{quoted_old_name}")}"
+          else
+            "NEW.#{quoted_new_name} := NEW.#{quoted_old_name}"
+          end
         end
 
         assignment_clauses.join(";\n  ")

data/lib/online_migrations/error_messages.rb

@@ -109,14 +109,8 @@ A safer approach is to:
 
 1. ignore the column:
 
-  class <%= model %> < <%= model_parent %>
-<% if ar_version >= 5 %>
+  class <%= model %> < ApplicationRecord
     self.ignored_columns = [\"<%= column_name %>\"]
-<% else %>
-    def self.columns
-      super.reject { |c| c.name == \"<%= column_name %>\" }
-    end
-<% end %>
   end
 
 2. deploy
@@ -157,13 +151,7 @@ It will use a combination of a VIEW and column aliasing to work with both column
 <% if enumerate_columns_in_select_statements %>
 5. Ignore old column
 
-<% if ar_version >= 5 %>
   self.ignored_columns = [:<%= column_name %>]
-<% else %>
-  def self.columns
-    super.reject { |c| c.name == \"<%= column_name %>\" }
-  end
-<% end %>
 
 6. Deploy
 7. Remove the column rename config from step 1
@@ -216,6 +204,8 @@ which will be passed to `add_column` when creating a new column, so you can over
 
     def up
       <%= backfill_code %>
+      # You can use `backfill_column_for_type_change_in_background` if want to
+      # backfill using background migrations.
     end
 
     def down
@@ -223,7 +213,10 @@ which will be passed to `add_column` when creating a new column, so you can over
     end
   end
 
-3. Copy indexes, foreign keys, check constraints, NOT NULL constraint, swap new column in place:
+3. Make sure your application works with values in both formats (when read from the database, converting
+during writes works automatically). For most column type changes, this does not need any updates in the app.
+4. Deploy
+5. Copy indexes, foreign keys, check constraints, NOT NULL constraint, swap new column in place:
 
   class Finalize<%= migration_name %> < <%= migration_parent %>
     disable_ddl_transaction!
@@ -233,8 +226,8 @@ which will be passed to `add_column` when creating a new column, so you can over
     end
   end
 
-4. Deploy
-5. Finally, if everything is working as expected, remove copy trigger and old column:
+6. Deploy
+7. Finally, if everything works as expected, remove copy trigger and old column:
 
   class Cleanup<%= migration_name %> < <%= migration_parent %>
     def up
@@ -246,7 +239,8 @@ which will be passed to `add_column` when creating a new column, so you can over
     end
   end
 
-6. Deploy",
+8. Remove changes from step 3, if any
+9. Deploy",
 
       change_column_default:
 "Partial writes are enabled, which can cause incorrect values
@@ -303,14 +297,8 @@ A safer approach is to:
 
 1. Ignore the column(s):
 
-  class <%= model %> < <%= model_parent %>
-<% if ar_version >= 5 %>
+  class <%= model %> < ApplicationRecord
     self.ignored_columns = <%= columns %>
-<% else %>
-    def self.columns
-      super.reject { |c| <%= columns %>.include?(c.name) }
-    end
-<% end %>
   end
 
 2. Deploy
@@ -504,7 +492,7 @@ execute call, so cannot help you here. Make really sure that what
 you're doing is safe before proceeding, then wrap it in a safety_assured { ... } block.",
 
       multiple_foreign_keys:
-"Adding multiple foreign keys in a single migration blocks reads and writes on all involved tables until migration is completed.
+"Adding multiple foreign keys in a single migration blocks writes on all involved tables until migration is completed.
 Avoid adding foreign key more than once per migration file, unless the source and target tables are identical.",
 
       drop_table_multiple_foreign_keys:

data/lib/online_migrations/foreign_keys_collector.rb

@@ -9,8 +9,8 @@ module OnlineMigrations
       @referenced_tables = Set.new
     end
 
-    def collect(&table_definition)
-      table_definition.call(self)
+    def collect
+      yield self
     end
 
     def foreign_key(to_table, **_options)

data/lib/online_migrations/indexes_collector.rb

@@ -12,8 +12,8 @@ module OnlineMigrations
       @indexes = []
     end
 
-    def collect(&table_definition)
-      table_definition.call(self)
+    def collect
+      yield self
     end
 
     def index(_column_name, **options)
@@ -21,7 +21,7 @@ module OnlineMigrations
     end
 
     def references(*_ref_names, **options)
-      index = options.fetch(:index) { Utils.ar_version >= 5.0 }
+      index = options.fetch(:index, true)
 
       if index
         using = index.is_a?(Hash) ? index[:using].to_s : nil

data/lib/online_migrations/lock_retrier.rb

@@ -96,9 +96,8 @@ module OnlineMigrations
         else
           yield
         end
-      # ActiveRecord::LockWaitTimeout can be used for Active Record 5.2+
-      rescue ActiveRecord::StatementInvalid => e
-        if lock_timeout_error?(e) && current_attempt <= attempts
+      rescue ActiveRecord::LockWaitTimeout
+        if current_attempt <= attempts
           current_delay = delay(current_attempt)
           Utils.say("Lock timeout. Retrying in #{current_delay} seconds...")
           sleep(current_delay)
@@ -122,10 +121,6 @@ module OnlineMigrations
       ensure
         connection.execute("SET lock_timeout TO #{connection.quote(prev_value)}")
       end
-
-      def lock_timeout_error?(error)
-        error.message.include?("canceling statement due to lock timeout")
-      end
   end
 
   # `LockRetrier` implementation that has a constant delay between tries
@@ -181,10 +176,10 @@ module OnlineMigrations
   #
   # @example
   #   # This will attempt 30 retries starting with delay of 10ms between each unsuccessful try, increasing exponentially
-  #   # up to the maximum delay of 1 minute and 50ms set as lock timeout for each try:
+  #   # up to the maximum delay of 1 minute and 200ms set as lock timeout for each try:
   #
   #   config.retrier = OnlineMigrations::ConstantLockRetrier.new(attempts: 30,
-  #       base_delay: 0.01.seconds, max_delay: 1.minute, lock_timeout: 0.05.seconds)
+  #       base_delay: 0.01.seconds, max_delay: 1.minute, lock_timeout: 0.2.seconds)
   #
   class ExponentialLockRetrier < LockRetrier
     # LockRetrier API implementation

data/lib/online_migrations/schema_cache.rb

@@ -28,9 +28,6 @@ module OnlineMigrations
     end
 
     def indexes(table_name)
-      # Available only in Active Record 6.0+
-      return if !defined?(super)
-
       if (renamed_table = renamed_table?(table_name))
         super(renamed_table)
       elsif renamed_column?(table_name)
@@ -109,9 +106,6 @@ module OnlineMigrations
     end
 
     def indexes(connection, table_name)
-      # Available only in Active Record 6.0+
-      return if !defined?(super)
-
       if (renamed_table = renamed_table?(connection, table_name))
         super(connection, renamed_table)
       elsif renamed_column?(connection, table_name)

data/lib/online_migrations/schema_dumper.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "delegate"
+
+module OnlineMigrations
+  module SchemaDumper
+    def initialize(connection, options = {})
+      if OnlineMigrations.config.alphabetize_schema
+        connection = WrappedConnection.new(connection)
+      end
+
+      super
+    end
+  end
+
+  class WrappedConnection < SimpleDelegator
+    def columns(table_name)
+      super.sort_by(&:name)
+    end
+  end
+end