safe-pg-migrations 3.0.0 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2e58bfc8a79c0b54a368c49a1fc720e33582220b472c81c3556158aa4d3e8f05
-  data.tar.gz: a112e1ade2a57568cdebee942d0a4f4f5eef1fcfb19db576d30c25eacf7fc04d
+  metadata.gz: 2048cd259a3da82cc4c7456c6c5fa0f9a04752886a864f9a6ed4e891c698c1e5
+  data.tar.gz: bd56fddb533483a264ee89691b7cbd23609a0f6517cc4fdf10f458e9af8152db
 SHA512:
-  metadata.gz: 7b28dcf7b252ebe311258b1c3ba4badfa0f365d40b53209e66b51c459a8fd960c3ceb452a378d0feb5785ec8f7632f575d53b9ec543e1de016729066d5e9ed42
-  data.tar.gz: 3338baf96b5d82f6055f6b59691fb42f0560e1d18c4a372685e9bc9ae40f55b58b1382bc8347e13b0d8628b754abae85f4d6c9d57437888efa334f9f82029e7b
+  metadata.gz: c3ee1fb684f03d3494ab950dd5e7a47be3e36fa710179fa25f2ffba86afb43d671f47d2f9a74f0b286d698181ab93a9d033894ecaba76d1b36976e54b897e788
+  data.tar.gz: 704b4f337c8f9dad758781faf51836889b272a161f08190578ae5d2dcb96fa4b5f6b6de019004f520e32ed9ba136ba9ba31d2c1a34c4df0780ba6a4bf0558d29

data/README.md

@@ -329,6 +329,10 @@ SafePgMigrations.config.safe_timeout = 5.seconds # Statement timeout used for al
 
 SafePgMigrations.config.lock_timeout = nil # Lock timeout used for all DDL operations except from CREATE / DROP INDEX. If not set, safe_timeout will be used with a deduction of 1% to ensure that the lock timeout is raised in priority
 
+SafePgMigrations.config.increase_lock_timeout_on_retry # Activate the lock timeout increase feature on retry if set to true. See max_lock_timeout_for_retry for more information.
+
+SafePgMigrations.config.max_lock_timeout_for_retry = 1.second # Max lock timeout for the retries for all DDL operations except from CREATE / DROP INDEX. Each retry will increase the lock_timeout (if increase_lock_timeout_on_retry option is set to true) by (max_lock_timeout_for_retry - lock_timeout) / max_tries
+
 SafePgMigrations.config.blocking_activity_logger_verbose = true # Outputs the raw blocking queries on timeout. When false, outputs information about the lock instead
 
 SafePgMigrations.config.sensitive_logger = nil # When given, sensitive data will be sent to this logger instead of the standard output. Must implement method `info`.

data/lib/safe-pg-migrations/base.rb

@@ -11,6 +11,7 @@ require 'safe-pg-migrations/plugins/verbose_sql_logger'
 require 'safe-pg-migrations/plugins/blocking_activity_logger'
 require 'safe-pg-migrations/plugins/statement_insurer/add_column'
 require 'safe-pg-migrations/plugins/statement_insurer/change_column_null'
+require 'safe-pg-migrations/plugins/statement_insurer/remove_column_index'
 require 'safe-pg-migrations/plugins/statement_insurer'
 require 'safe-pg-migrations/plugins/statement_retrier'
 require 'safe-pg-migrations/plugins/idempotent_statements'

data/lib/safe-pg-migrations/configuration.rb

@@ -5,50 +5,66 @@ require 'active_support/core_ext/numeric/time'
 module SafePgMigrations
   class Configuration
     attr_accessor(*%i[
+                    backfill_batch_size
+                    backfill_pause
                     blocking_activity_logger_margin
                     blocking_activity_logger_verbose
                     default_value_backfill_threshold
-                    backfill_batch_size
-                    backfill_pause
-                    retry_delay
+                    increase_lock_timeout_on_retry
                     max_tries
+                    retry_delay
                     sensitive_logger
                   ])
-    attr_reader :lock_timeout, :safe_timeout
+    attr_reader :lock_timeout, :safe_timeout, :max_lock_timeout_for_retry
 
     def initialize
-      self.default_value_backfill_threshold = nil
-      self.safe_timeout = 5.seconds
-      self.lock_timeout = nil
-      self.blocking_activity_logger_margin = 1.second
-      self.blocking_activity_logger_verbose = true
       self.backfill_batch_size = 100_000
       self.backfill_pause = 0.5.second
-      self.retry_delay = 1.minute
+      self.blocking_activity_logger_margin = 1.second
+      self.blocking_activity_logger_verbose = true
+      self.default_value_backfill_threshold = nil
+      self.increase_lock_timeout_on_retry = false
+      self.lock_timeout = nil
+      self.max_lock_timeout_for_retry = 1.second
       self.max_tries = 5
+      self.retry_delay = 1.minute
+      self.safe_timeout = 5.seconds
       self.sensitive_logger = nil
     end
 
     def lock_timeout=(value)
       raise 'Setting lock timeout to 0 disables the lock timeout and is dangerous' if value == 0.seconds
 
-      unless value.nil? || value < safe_timeout
-        raise ArgumentError, "Lock timeout (#{value}) cannot be greater than safe timeout (#{safe_timeout})"
+      unless value.nil? || (value < safe_timeout && value <= max_lock_timeout_for_retry)
+        raise ArgumentError, "Lock timeout (#{value}) cannot be greater than the safe timeout (#{safe_timeout}) or the
+                              max lock timeout for retry (#{max_lock_timeout_for_retry})"
       end
 
       @lock_timeout = value
     end
 
     def safe_timeout=(value)
-      raise 'Setting safe timeout to 0 disables the safe timeout and is dangerous' unless value
+      unless value && value > 0.seconds
+        raise 'Setting safe timeout to 0 or nil disables the safe timeout and is dangerous'
+      end
 
-      unless lock_timeout.nil? || value > lock_timeout
-        raise ArgumentError, "Safe timeout (#{value}) cannot be less than lock timeout (#{lock_timeout})"
+      unless lock_timeout.nil? || (value > lock_timeout && value >= max_lock_timeout_for_retry)
+        raise ArgumentError, "Safe timeout (#{value}) cannot be lower than the lock timeout (#{lock_timeout}) or the
+                              max lock timeout for retry (#{max_lock_timeout_for_retry})"
       end
 
       @safe_timeout = value
     end
 
+    def max_lock_timeout_for_retry=(value)
+      unless lock_timeout.nil? || (value >= lock_timeout && value <= safe_timeout)
+        raise ArgumentError, "Max lock timeout for retry (#{value}) cannot be lower than the lock timeout
+                              (#{lock_timeout}) and greater than the safe timeout (#{safe_timeout})"
+      end
+
+      @max_lock_timeout_for_retry = value
+    end
+
     def pg_statement_timeout
       pg_duration safe_timeout
     end

data/lib/safe-pg-migrations/helpers/batch_over.rb

@@ -19,11 +19,11 @@ module SafePgMigrations
       def next_batch
         return if endless?
 
-        first = next_scope.take
+        first = next_scope.select(primary_key).take
 
         return unless first
 
-        last = next_scope.offset(@of).take
+        last = next_scope.select(primary_key).offset(@of).take
 
         first_key = first[primary_key]
         last_key = last.nil? ? nil : last[primary_key]

data/lib/safe-pg-migrations/plugins/statement_insurer/add_column.rb

@@ -21,7 +21,7 @@ module SafePgMigrations
         null = options.delete(:null)
 
         Helpers::Logger.say_method_call(:add_column, table_name, column_name, type, options)
-        super table_name, column_name, type, **options
+        super(table_name, column_name, type, **options)
 
         Helpers::Logger.say_method_call(:change_column_default, table_name, column_name, default)
         change_column_default(table_name, column_name, default)

data/lib/safe-pg-migrations/plugins/statement_insurer/change_column_null.rb

@@ -16,7 +16,7 @@ module SafePgMigrations
         add_check_constraint table_name, expression, name: constraint_name
 
         Helpers::Logger.say_method_call :change_column_null, table_name, column_name, false
-        super table_name, column_name, false
+        super(table_name, column_name, false)
 
         return unless should_remove_constraint? default_name, constraint_name
 
@@ -27,7 +27,7 @@ module SafePgMigrations
       private
 
       def check_constraint_by_expression(table_name, expression)
-        check_constraints(table_name).detect { |check_constraint| check_constraint.expression = expression }
+        check_constraints(table_name).detect { |check_constraint| check_constraint.expression == expression }
       end
 
       def should_create_constraint?(default, null)

data/lib/safe-pg-migrations/plugins/statement_insurer/remove_column_index.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module SafePgMigrations
+  module StatementInsurer
+    module RemoveColumnIndex
+      def remove_column_with_composite_index(table, column)
+        existing_indexes = indexes(table).select { |index|
+          index.columns.size > 1 && index.columns.include?(column.to_s)
+        }
+
+        return unless existing_indexes.any?
+
+        error_message = <<~ERROR
+          Cannot drop column #{column} from table #{table} because composite index(es): #{existing_indexes.map(&:name).join(', ')} is/are present.
+          If they are still required, create the index(es) without #{column} before dropping the existing index(es).
+          Then you will be able to drop the column.
+        ERROR
+
+        raise StandardError, error_message
+      end
+    end
+  end
+end

data/lib/safe-pg-migrations/plugins/statement_insurer.rb

@@ -5,6 +5,7 @@ module SafePgMigrations
     include Helpers::SessionSettingManagement
     include AddColumn
     include ChangeColumnNull
+    include RemoveColumnIndex
 
     def validate_check_constraint(table_name, **options)
       Helpers::Logger.say_method_call :validate_check_constraint, table_name, **options
@@ -19,7 +20,7 @@ module SafePgMigrations
 
       Helpers::Logger.say_method_call :add_check_constraint, table_name, expression, **options,
         validate: false
-      super table_name, expression, **options, validate: false
+      super(table_name, expression, **options, validate: false)
 
       return unless options.fetch(:validate, true)
 
@@ -42,7 +43,7 @@ module SafePgMigrations
       validate_foreign_key from_table, sub_options.present? ? nil : to_table, **sub_options
     end
 
-    def create_table(*)
+    def create_table(table_name, **options)
       super do |td|
         yield td if block_given?
         td.indexes.map! do |key, index_options|
@@ -74,6 +75,7 @@ module SafePgMigrations
       foreign_key = foreign_key_for(table_name, column: column_name)
 
       remove_foreign_key(table_name, name: foreign_key.name) if foreign_key
+      remove_column_with_composite_index(table_name, column_name)
       super
     end
 

data/lib/safe-pg-migrations/plugins/statement_retrier.rb

@@ -13,20 +13,46 @@ module SafePgMigrations
     private
 
     def retry_if_lock_timeout
-      remaining_tries = SafePgMigrations.config.max_tries
+      initial_lock_timeout = SafePgMigrations.config.lock_timeout
+      number_of_retries = 0
       begin
-        remaining_tries -= 1
+        number_of_retries += 1
         yield
       rescue ActiveRecord::LockWaitTimeout
-        raise if transaction_open? # Retrying is useless if we're inside a transaction.
-        raise unless remaining_tries > 0
+        # Retrying is useless if we're inside a transaction.
+        if transaction_open? || number_of_retries >= SafePgMigrations.config.max_tries
+          SafePgMigrations.config.lock_timeout = initial_lock_timeout
+          raise
+        end
 
         retry_delay = SafePgMigrations.config.retry_delay
         Helpers::Logger.say "Retrying in #{retry_delay} seconds...", sub_item: true
+
+        if SafePgMigrations.config.increase_lock_timeout_on_retry && !SafePgMigrations.config.lock_timeout.nil?
+          increase_lock_timeout
+        end
+
         sleep retry_delay
         Helpers::Logger.say 'Retrying now.', sub_item: true
         retry
       end
     end
+
+    def increase_lock_timeout
+      Helpers::Logger.say "  Increasing the lock timeout... Currently set to #{SafePgMigrations.config.lock_timeout}",
+                          sub_item: true
+      SafePgMigrations.config.lock_timeout = (SafePgMigrations.config.lock_timeout + lock_timeout_step)
+      unless SafePgMigrations.config.lock_timeout < SafePgMigrations.config.max_lock_timeout_for_retry
+        SafePgMigrations.config.lock_timeout = SafePgMigrations.config.max_lock_timeout_for_retry
+      end
+      Helpers::Logger.say "  Lock timeout is now set to #{SafePgMigrations.config.lock_timeout}", sub_item: true
+    end
+
+    def lock_timeout_step
+      max_lock_timeout_for_retry = SafePgMigrations.config.max_lock_timeout_for_retry
+      lock_timeout = SafePgMigrations.config.lock_timeout
+      max_tries = SafePgMigrations.config.max_tries
+      @lock_timeout_step ||= (max_lock_timeout_for_retry - lock_timeout) / (max_tries - 1)
+    end
   end
 end

data/lib/safe-pg-migrations/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SafePgMigrations
-  VERSION = '3.0.0'
+  VERSION = '3.1.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: safe-pg-migrations
 version: !ruby/object:Gem::Version
-  version: 3.0.0
+  version: 3.1.0
 platform: ruby
 authors:
 - Matthieu Prat
@@ -11,7 +11,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-12-11 00:00:00.000000000 Z
+date: 2024-05-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -65,6 +65,7 @@ files:
 - lib/safe-pg-migrations/plugins/statement_insurer.rb
 - lib/safe-pg-migrations/plugins/statement_insurer/add_column.rb
 - lib/safe-pg-migrations/plugins/statement_insurer/change_column_null.rb
+- lib/safe-pg-migrations/plugins/statement_insurer/remove_column_index.rb
 - lib/safe-pg-migrations/plugins/statement_retrier.rb
 - lib/safe-pg-migrations/plugins/strong_migrations_integration.rb
 - lib/safe-pg-migrations/plugins/useless_statements_logger.rb
@@ -98,7 +99,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.4.19
 signing_key:
 specification_version: 4
 summary: Make your PG migrations safe.