safe-pg-migrations 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d806334c473708774d2180dbe91d2483b49f0d27b0a223a2973bc95d986d4030
-  data.tar.gz: 9b5c2d329d5cc1bd12c80944420e20eb5454e5b0a0f7fda2d6ec3fc3c3a5e042
+  metadata.gz: e6c0ad1795c94252929d907f94e3cf65617ca9a353414d031fb74a852ee294ea
+  data.tar.gz: f2d1045f55ff1896ef045edb4fe2868b7f26bc4e1b7cc542907eaab08da91075
 SHA512:
-  metadata.gz: 6786e68cce9dbd91a4633c80526a1fd63f6fbfa335d6e2b53d29b71ba0b566f9bb32169745bb13f89a059b82b0c0b21585a9bffdb9ebc751aa0e4a467f0f7934
-  data.tar.gz: 299a69f1d9413279fd61908a7d8f6eb65bd4c9912abcc1101ca8acbfee39c40d33086efa28a9f2133c0679c1ad9cdbbfe3ce84338f2175ad882b7d1ace707654
+  metadata.gz: 3b32973e6920a5db9d7d48440cc5ac09fbdd511539dcf3ec436584584d6a9b501a1027a212d991a27e47ae9a20e471cabc6d705e42f6324eb79eb1b529186f1f
+  data.tar.gz: 0c006975e25f1c318caa3375a05c1aa43a45c50813ef6b03a4d4d7d842b5c5a2188243744a8f4e769196c39ef6ca162c693a22b48dec1c38f13aa4b1d61bfc80

data/README.md

@@ -122,7 +122,7 @@ Note: the addition of the not null constraint may timeout. In that case, you may
 
 </details>
 
-<details><summary>Safe <code>add_index</code> and <code>remove_index</code></summary>
+<details><summary id="safe_add_remove_index">Safe <code>add_index</code> and <code>remove_index</code></summary>
 
 Creating an index requires a `SHARE` lock on the target table which blocks all write on the table while the index is created (which can take some time on a large table). This is usually not practical in a live environment. Thus, **Safe PG Migrations** ensures indexes are created concurrently.
 
@@ -135,7 +135,7 @@ If you still get lock timeout while adding / removing indexes, it might be for o
 
 </details>
 
-<details><summary>safe <code>add_foreign_key</code> (and <code>add_reference</code>)</summary>
+<details><summary id="safe_add_foreign_key">safe <code>add_foreign_key</code> (and <code>add_reference</code>)</summary>
 
 Adding a foreign key requires a `SHARE ROW EXCLUSIVE` lock, which **prevent writing in the tables** while the migration is running.
 
@@ -206,9 +206,7 @@ So you can actually check that the `CREATE INDEX` statement will be performed co
 ```ruby
 SafePgMigrations.config.safe_timeout = 5.seconds # Lock and statement timeout used for all DDL operations except from CREATE / DROP INDEX
 
-SafePgMigrations.config.index_lock_timeout = 30.seconds # Lock timeout used for CREATE / DROP INDEX
-
-SafePgMigrations.config.blocking_activity_logger_margin = 1.second # Delay to output blocking queries before timeout. Must be smaller than safe_timeout and index_lock_timeout
+SafePgMigrations.config.blocking_activity_logger_margin = 1.second # Delay to output blocking queries before timeout. Must be shorter than safe_timeout
 
 SafePgMigrations.config.batch_size = 1000 # Size of the batches used for backfilling when adding a column with a default value pre-PG11
 

data/lib/safe-pg-migrations/base.rb

@@ -6,6 +6,7 @@ require 'safe-pg-migrations/plugins/blocking_activity_logger'
 require 'safe-pg-migrations/plugins/statement_insurer'
 require 'safe-pg-migrations/plugins/statement_retrier'
 require 'safe-pg-migrations/plugins/idem_potent_statements'
+require 'safe-pg-migrations/plugins/useless_statements_logger'
 
 module SafePgMigrations
   # Order matters: the bottom-most plugin will have precedence
@@ -14,6 +15,7 @@ module SafePgMigrations
     IdemPotentStatements,
     StatementRetrier,
     StatementInsurer,
+    UselessStatementsLogger,
   ].freeze
 
   class << self
@@ -78,6 +80,7 @@ module SafePgMigrations
     end
 
     def disable_ddl_transaction
+      UselessStatementsLogger.warn_useless '`disable_ddl_transaction`' if super
       true
     end
 

data/lib/safe-pg-migrations/configuration.rb

@@ -5,7 +5,6 @@ require 'active_support/core_ext/numeric/time'
 module SafePgMigrations
   class Configuration
     attr_accessor :safe_timeout
-    attr_accessor :index_lock_timeout
     attr_accessor :blocking_activity_logger_margin
     attr_accessor :batch_size
     attr_accessor :retry_delay
@@ -13,7 +12,6 @@ module SafePgMigrations
 
     def initialize
       self.safe_timeout = 5.seconds
-      self.index_lock_timeout = 30.seconds
       self.blocking_activity_logger_margin = 1.second
       self.batch_size = 1000
       self.retry_delay = 1.minute
@@ -24,10 +22,6 @@ module SafePgMigrations
       pg_duration(safe_timeout)
     end
 
-    def pg_index_lock_timeout
-      pg_duration(index_lock_timeout)
-    end
-
     def pg_duration(duration)
       value, unit = duration.integer? ? [duration, 's'] : [(duration * 1000).to_i, 'ms']
       "#{value}#{unit}"

data/lib/safe-pg-migrations/plugins/blocking_activity_logger.rb

@@ -3,7 +3,7 @@
 module SafePgMigrations
   module BlockingActivityLogger
     SELECT_BLOCKING_QUERIES_SQL = <<~SQL.squish
-      SELECT blocking_activity.query
+      SELECT blocking_activity.query, blocked_activity.xact_start as start
       FROM pg_catalog.pg_locks           blocked_locks
       JOIN pg_catalog.pg_stat_activity   blocked_activity
         ON blocked_activity.pid = blocked_locks.pid
@@ -25,32 +25,23 @@ module SafePgMigrations
     SQL
 
     %i[
-      add_column remove_column add_foreign_key remove_foreign_key change_column_default
-      change_column_null create_table add_index remove_index
+      add_column remove_column add_foreign_key remove_foreign_key change_column_default change_column_null create_table
     ].each do |method|
       define_method method do |*args, &block|
-        log_blocking_queries(method) { super(*args, &block) }
+        log_blocking_queries { super(*args, &block) }
       end
     end
 
     private
 
-    def delay_before_logging(method)
-      timeout_delay =
-        if %i[add_index remove_index].include?(method)
-          SafePgMigrations.config.index_lock_timeout
-        else
-          SafePgMigrations.config.safe_timeout
-        end
-
-      timeout_delay - SafePgMigrations.config.blocking_activity_logger_margin
-    end
+    def log_blocking_queries
+      delay_before_logging =
+        SafePgMigrations.config.safe_timeout - SafePgMigrations.config.blocking_activity_logger_margin
 
-    def log_blocking_queries(method)
       blocking_queries_retriever_thread =
         Thread.new do
-          sleep delay_before_logging(method)
-          SafePgMigrations.alternate_connection.query_values(SELECT_BLOCKING_QUERIES_SQL % raw_connection.backend_pid)
+          sleep delay_before_logging
+          SafePgMigrations.alternate_connection.query(SELECT_BLOCKING_QUERIES_SQL % raw_connection.backend_pid)
         end
 
       yield
@@ -75,7 +66,7 @@ module SafePgMigrations
           "Statement was being blocked by the following #{'query'.pluralize(queries.size)}:", true
         )
         SafePgMigrations.say '', true
-        queries.each { |query| SafePgMigrations.say "  #{query}", true }
+        queries.each { |query, start_time| SafePgMigrations.say "#{format_start_time start_time}:  #{query}", true }
         SafePgMigrations.say(
           'Beware, some of those queries might run in a transaction. In this case the locking query might be '\
           'located elsewhere in the transaction',
@@ -86,5 +77,10 @@ module SafePgMigrations
 
       raise
     end
+
+    def format_start_time(start_time, reference_time = Time.now)
+      duration = (reference_time - start_time).round
+      "transaction started #{duration} #{'second'.pluralize(duration)} ago"
+    end
   end
 end

data/lib/safe-pg-migrations/plugins/idem_potent_statements.rb

@@ -43,12 +43,19 @@ module SafePgMigrations
     end
 
     def create_table(table_name, comment: nil, **options)
-      return super unless table_exists?(table_name)
+      return super if options[:force] || !table_exists?(table_name)
 
-      SafePgMigrations.say(
-        "/!\\ Table '#{table_name}' already exists. Skipping statement.",
-        true
-      )
+      SafePgMigrations.say "/!\\ Table '#{table_name}' already exists.", true
+
+      td = create_table_definition(table_name, **options)
+
+      yield td if block_given?
+
+      SafePgMigrations.say(td.indexes.empty? ? '-- Skipping statement' : '-- Creating indexes', true)
+
+      td.indexes.each do |column_name, index_options|
+        add_index(table_name, column_name, index_options)
+      end
     end
 
     private

data/lib/safe-pg-migrations/plugins/statement_insurer.rb

@@ -50,7 +50,7 @@ module SafePgMigrations
       options[:algorithm] = :concurrently
       SafePgMigrations.say_method_call(:add_index, table_name, column_name, options)
 
-      with_index_timeouts { super }
+      without_timeout { super }
     end
 
     def remove_index(table_name, options = {})
@@ -58,7 +58,7 @@ module SafePgMigrations
       options[:algorithm] = :concurrently
       SafePgMigrations.say_method_call(:remove_index, table_name, options)
 
-      with_index_timeouts { super }
+      without_timeout { super }
     end
 
     def backfill_column_default(table_name, column_name)
@@ -99,12 +99,12 @@ module SafePgMigrations
       with_setting(:statement_timeout, 0) { yield }
     end
 
-    def with_index_timeouts
-      without_statement_timeout do
-        with_setting(:lock_timeout, SafePgMigrations.config.pg_index_lock_timeout) do
-          yield
-        end
-      end
+    def without_lock_timeout
+      with_setting(:lock_timeout, 0) { yield }
+    end
+
+    def without_timeout
+      without_statement_timeout { without_lock_timeout { yield } }
     end
   end
 end

data/lib/safe-pg-migrations/plugins/statement_retrier.rb

@@ -3,8 +3,7 @@
 module SafePgMigrations
   module StatementRetrier
     RETRIABLE_SCHEMA_STATEMENTS = %i[
-      add_column add_foreign_key remove_foreign_key change_column_default
-      change_column_null add_index remove_index remove_column
+      add_column add_foreign_key remove_foreign_key change_column_default change_column_null remove_column
     ].freeze
 
     RETRIABLE_SCHEMA_STATEMENTS.each do |method|

data/lib/safe-pg-migrations/plugins/useless_statements_logger.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module SafePgMigrations
+  module UselessStatementsLogger
+    def self.warn_useless(action, link = nil, *args)
+      SafePgMigrations.say "/!\\ No need to explicitly use #{action}, safe-pg-migrations does it for you", *args
+      SafePgMigrations.say "\t see #{link} for more details", *args if link
+    end
+
+    def add_index(*, **options)
+      warn_for_index(**options)
+      super
+    end
+
+    def remove_index(table_name, options = {})
+      warn_for_index(options) if options.is_a? Hash
+      super
+    end
+
+    def add_foreign_key(*, **options)
+      if options[:validate] == false
+        UselessStatementsLogger.warn_useless '`validate: :false`', 'https://github.com/doctolib/safe-pg-migrations#safe_add_foreign_key'
+      end
+      super
+    end
+
+    def warn_for_index(**options)
+      return unless options[:algorithm] == :concurrently
+
+      UselessStatementsLogger.warn_useless '`algorithm: :concurrently`', 'https://github.com/doctolib/safe-pg-migrations#safe_add_remove_index'
+    end
+  end
+end

data/lib/safe-pg-migrations/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SafePgMigrations
-  VERSION = '1.1.0'
+  VERSION = '1.2.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: safe-pg-migrations
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Matthieu Prat
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-11-03 00:00:00.000000000 Z
+date: 2020-11-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -166,6 +166,7 @@ files:
 - lib/safe-pg-migrations/plugins/idem_potent_statements.rb
 - lib/safe-pg-migrations/plugins/statement_insurer.rb
 - lib/safe-pg-migrations/plugins/statement_retrier.rb
+- lib/safe-pg-migrations/plugins/useless_statements_logger.rb
 - lib/safe-pg-migrations/plugins/verbose_sql_logger.rb
 - lib/safe-pg-migrations/railtie.rb
 - lib/safe-pg-migrations/version.rb