safe-pg-migrations 3.0.0 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 2e58bfc8a79c0b54a368c49a1fc720e33582220b472c81c3556158aa4d3e8f05
4
- data.tar.gz: a112e1ade2a57568cdebee942d0a4f4f5eef1fcfb19db576d30c25eacf7fc04d
3
+ metadata.gz: 2048cd259a3da82cc4c7456c6c5fa0f9a04752886a864f9a6ed4e891c698c1e5
4
+ data.tar.gz: bd56fddb533483a264ee89691b7cbd23609a0f6517cc4fdf10f458e9af8152db
5
5
  SHA512:
6
- metadata.gz: 7b28dcf7b252ebe311258b1c3ba4badfa0f365d40b53209e66b51c459a8fd960c3ceb452a378d0feb5785ec8f7632f575d53b9ec543e1de016729066d5e9ed42
7
- data.tar.gz: 3338baf96b5d82f6055f6b59691fb42f0560e1d18c4a372685e9bc9ae40f55b58b1382bc8347e13b0d8628b754abae85f4d6c9d57437888efa334f9f82029e7b
6
+ metadata.gz: c3ee1fb684f03d3494ab950dd5e7a47be3e36fa710179fa25f2ffba86afb43d671f47d2f9a74f0b286d698181ab93a9d033894ecaba76d1b36976e54b897e788
7
+ data.tar.gz: 704b4f337c8f9dad758781faf51836889b272a161f08190578ae5d2dcb96fa4b5f6b6de019004f520e32ed9ba136ba9ba31d2c1a34c4df0780ba6a4bf0558d29
data/README.md CHANGED
@@ -329,6 +329,10 @@ SafePgMigrations.config.safe_timeout = 5.seconds # Statement timeout used for al
329
329
 
330
330
  SafePgMigrations.config.lock_timeout = nil # Lock timeout used for all DDL operations except from CREATE / DROP INDEX. If not set, safe_timeout will be used with a deduction of 1% to ensure that the lock timeout is raised in priority
331
331
 
332
+ SafePgMigrations.config.increase_lock_timeout_on_retry # Activate the lock timeout increase feature on retry if set to true. See max_lock_timeout_for_retry for more information.
333
+
334
+ SafePgMigrations.config.max_lock_timeout_for_retry = 1.second # Max lock timeout for the retries for all DDL operations except from CREATE / DROP INDEX. Each retry will increase the lock_timeout (if increase_lock_timeout_on_retry option is set to true) by (max_lock_timeout_for_retry - lock_timeout) / max_tries
335
+
332
336
  SafePgMigrations.config.blocking_activity_logger_verbose = true # Outputs the raw blocking queries on timeout. When false, outputs information about the lock instead
333
337
 
334
338
  SafePgMigrations.config.sensitive_logger = nil # When given, sensitive data will be sent to this logger instead of the standard output. Must implement method `info`.
@@ -11,6 +11,7 @@ require 'safe-pg-migrations/plugins/verbose_sql_logger'
11
11
  require 'safe-pg-migrations/plugins/blocking_activity_logger'
12
12
  require 'safe-pg-migrations/plugins/statement_insurer/add_column'
13
13
  require 'safe-pg-migrations/plugins/statement_insurer/change_column_null'
14
+ require 'safe-pg-migrations/plugins/statement_insurer/remove_column_index'
14
15
  require 'safe-pg-migrations/plugins/statement_insurer'
15
16
  require 'safe-pg-migrations/plugins/statement_retrier'
16
17
  require 'safe-pg-migrations/plugins/idempotent_statements'
@@ -5,50 +5,66 @@ require 'active_support/core_ext/numeric/time'
5
5
  module SafePgMigrations
6
6
  class Configuration
7
7
  attr_accessor(*%i[
8
+ backfill_batch_size
9
+ backfill_pause
8
10
  blocking_activity_logger_margin
9
11
  blocking_activity_logger_verbose
10
12
  default_value_backfill_threshold
11
- backfill_batch_size
12
- backfill_pause
13
- retry_delay
13
+ increase_lock_timeout_on_retry
14
14
  max_tries
15
+ retry_delay
15
16
  sensitive_logger
16
17
  ])
17
- attr_reader :lock_timeout, :safe_timeout
18
+ attr_reader :lock_timeout, :safe_timeout, :max_lock_timeout_for_retry
18
19
 
19
20
  def initialize
20
- self.default_value_backfill_threshold = nil
21
- self.safe_timeout = 5.seconds
22
- self.lock_timeout = nil
23
- self.blocking_activity_logger_margin = 1.second
24
- self.blocking_activity_logger_verbose = true
25
21
  self.backfill_batch_size = 100_000
26
22
  self.backfill_pause = 0.5.second
27
- self.retry_delay = 1.minute
23
+ self.blocking_activity_logger_margin = 1.second
24
+ self.blocking_activity_logger_verbose = true
25
+ self.default_value_backfill_threshold = nil
26
+ self.increase_lock_timeout_on_retry = false
27
+ self.lock_timeout = nil
28
+ self.max_lock_timeout_for_retry = 1.second
28
29
  self.max_tries = 5
30
+ self.retry_delay = 1.minute
31
+ self.safe_timeout = 5.seconds
29
32
  self.sensitive_logger = nil
30
33
  end
31
34
 
32
35
  def lock_timeout=(value)
33
36
  raise 'Setting lock timeout to 0 disables the lock timeout and is dangerous' if value == 0.seconds
34
37
 
35
- unless value.nil? || value < safe_timeout
36
- raise ArgumentError, "Lock timeout (#{value}) cannot be greater than safe timeout (#{safe_timeout})"
38
+ unless value.nil? || (value < safe_timeout && value <= max_lock_timeout_for_retry)
39
+ raise ArgumentError, "Lock timeout (#{value}) cannot be greater than the safe timeout (#{safe_timeout}) or the
40
+ max lock timeout for retry (#{max_lock_timeout_for_retry})"
37
41
  end
38
42
 
39
43
  @lock_timeout = value
40
44
  end
41
45
 
42
46
  def safe_timeout=(value)
43
- raise 'Setting safe timeout to 0 disables the safe timeout and is dangerous' unless value
47
+ unless value && value > 0.seconds
48
+ raise 'Setting safe timeout to 0 or nil disables the safe timeout and is dangerous'
49
+ end
44
50
 
45
- unless lock_timeout.nil? || value > lock_timeout
46
- raise ArgumentError, "Safe timeout (#{value}) cannot be less than lock timeout (#{lock_timeout})"
51
+ unless lock_timeout.nil? || (value > lock_timeout && value >= max_lock_timeout_for_retry)
52
+ raise ArgumentError, "Safe timeout (#{value}) cannot be lower than the lock timeout (#{lock_timeout}) or the
53
+ max lock timeout for retry (#{max_lock_timeout_for_retry})"
47
54
  end
48
55
 
49
56
  @safe_timeout = value
50
57
  end
51
58
 
59
+ def max_lock_timeout_for_retry=(value)
60
+ unless lock_timeout.nil? || (value >= lock_timeout && value <= safe_timeout)
61
+ raise ArgumentError, "Max lock timeout for retry (#{value}) cannot be lower than the lock timeout
62
+ (#{lock_timeout}) and greater than the safe timeout (#{safe_timeout})"
63
+ end
64
+
65
+ @max_lock_timeout_for_retry = value
66
+ end
67
+
52
68
  def pg_statement_timeout
53
69
  pg_duration safe_timeout
54
70
  end
@@ -19,11 +19,11 @@ module SafePgMigrations
19
19
  def next_batch
20
20
  return if endless?
21
21
 
22
- first = next_scope.take
22
+ first = next_scope.select(primary_key).take
23
23
 
24
24
  return unless first
25
25
 
26
- last = next_scope.offset(@of).take
26
+ last = next_scope.select(primary_key).offset(@of).take
27
27
 
28
28
  first_key = first[primary_key]
29
29
  last_key = last.nil? ? nil : last[primary_key]
@@ -21,7 +21,7 @@ module SafePgMigrations
21
21
  null = options.delete(:null)
22
22
 
23
23
  Helpers::Logger.say_method_call(:add_column, table_name, column_name, type, options)
24
- super table_name, column_name, type, **options
24
+ super(table_name, column_name, type, **options)
25
25
 
26
26
  Helpers::Logger.say_method_call(:change_column_default, table_name, column_name, default)
27
27
  change_column_default(table_name, column_name, default)
@@ -16,7 +16,7 @@ module SafePgMigrations
16
16
  add_check_constraint table_name, expression, name: constraint_name
17
17
 
18
18
  Helpers::Logger.say_method_call :change_column_null, table_name, column_name, false
19
- super table_name, column_name, false
19
+ super(table_name, column_name, false)
20
20
 
21
21
  return unless should_remove_constraint? default_name, constraint_name
22
22
 
@@ -27,7 +27,7 @@ module SafePgMigrations
27
27
  private
28
28
 
29
29
  def check_constraint_by_expression(table_name, expression)
30
- check_constraints(table_name).detect { |check_constraint| check_constraint.expression = expression }
30
+ check_constraints(table_name).detect { |check_constraint| check_constraint.expression == expression }
31
31
  end
32
32
 
33
33
  def should_create_constraint?(default, null)
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SafePgMigrations
4
+ module StatementInsurer
5
+ module RemoveColumnIndex
6
+ def remove_column_with_composite_index(table, column)
7
+ existing_indexes = indexes(table).select { |index|
8
+ index.columns.size > 1 && index.columns.include?(column.to_s)
9
+ }
10
+
11
+ return unless existing_indexes.any?
12
+
13
+ error_message = <<~ERROR
14
+ Cannot drop column #{column} from table #{table} because composite index(es): #{existing_indexes.map(&:name).join(', ')} is/are present.
15
+ If they are still required, create the index(es) without #{column} before dropping the existing index(es).
16
+ Then you will be able to drop the column.
17
+ ERROR
18
+
19
+ raise StandardError, error_message
20
+ end
21
+ end
22
+ end
23
+ end
@@ -5,6 +5,7 @@ module SafePgMigrations
5
5
  include Helpers::SessionSettingManagement
6
6
  include AddColumn
7
7
  include ChangeColumnNull
8
+ include RemoveColumnIndex
8
9
 
9
10
  def validate_check_constraint(table_name, **options)
10
11
  Helpers::Logger.say_method_call :validate_check_constraint, table_name, **options
@@ -19,7 +20,7 @@ module SafePgMigrations
19
20
 
20
21
  Helpers::Logger.say_method_call :add_check_constraint, table_name, expression, **options,
21
22
  validate: false
22
- super table_name, expression, **options, validate: false
23
+ super(table_name, expression, **options, validate: false)
23
24
 
24
25
  return unless options.fetch(:validate, true)
25
26
 
@@ -42,7 +43,7 @@ module SafePgMigrations
42
43
  validate_foreign_key from_table, sub_options.present? ? nil : to_table, **sub_options
43
44
  end
44
45
 
45
- def create_table(*)
46
+ def create_table(table_name, **options)
46
47
  super do |td|
47
48
  yield td if block_given?
48
49
  td.indexes.map! do |key, index_options|
@@ -74,6 +75,7 @@ module SafePgMigrations
74
75
  foreign_key = foreign_key_for(table_name, column: column_name)
75
76
 
76
77
  remove_foreign_key(table_name, name: foreign_key.name) if foreign_key
78
+ remove_column_with_composite_index(table_name, column_name)
77
79
  super
78
80
  end
79
81
 
@@ -13,20 +13,46 @@ module SafePgMigrations
13
13
  private
14
14
 
15
15
  def retry_if_lock_timeout
16
- remaining_tries = SafePgMigrations.config.max_tries
16
+ initial_lock_timeout = SafePgMigrations.config.lock_timeout
17
+ number_of_retries = 0
17
18
  begin
18
- remaining_tries -= 1
19
+ number_of_retries += 1
19
20
  yield
20
21
  rescue ActiveRecord::LockWaitTimeout
21
- raise if transaction_open? # Retrying is useless if we're inside a transaction.
22
- raise unless remaining_tries > 0
22
+ # Retrying is useless if we're inside a transaction.
23
+ if transaction_open? || number_of_retries >= SafePgMigrations.config.max_tries
24
+ SafePgMigrations.config.lock_timeout = initial_lock_timeout
25
+ raise
26
+ end
23
27
 
24
28
  retry_delay = SafePgMigrations.config.retry_delay
25
29
  Helpers::Logger.say "Retrying in #{retry_delay} seconds...", sub_item: true
30
+
31
+ if SafePgMigrations.config.increase_lock_timeout_on_retry && !SafePgMigrations.config.lock_timeout.nil?
32
+ increase_lock_timeout
33
+ end
34
+
26
35
  sleep retry_delay
27
36
  Helpers::Logger.say 'Retrying now.', sub_item: true
28
37
  retry
29
38
  end
30
39
  end
40
+
41
+ def increase_lock_timeout
42
+ Helpers::Logger.say " Increasing the lock timeout... Currently set to #{SafePgMigrations.config.lock_timeout}",
43
+ sub_item: true
44
+ SafePgMigrations.config.lock_timeout = (SafePgMigrations.config.lock_timeout + lock_timeout_step)
45
+ unless SafePgMigrations.config.lock_timeout < SafePgMigrations.config.max_lock_timeout_for_retry
46
+ SafePgMigrations.config.lock_timeout = SafePgMigrations.config.max_lock_timeout_for_retry
47
+ end
48
+ Helpers::Logger.say " Lock timeout is now set to #{SafePgMigrations.config.lock_timeout}", sub_item: true
49
+ end
50
+
51
+ def lock_timeout_step
52
+ max_lock_timeout_for_retry = SafePgMigrations.config.max_lock_timeout_for_retry
53
+ lock_timeout = SafePgMigrations.config.lock_timeout
54
+ max_tries = SafePgMigrations.config.max_tries
55
+ @lock_timeout_step ||= (max_lock_timeout_for_retry - lock_timeout) / (max_tries - 1)
56
+ end
31
57
  end
32
58
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module SafePgMigrations
4
- VERSION = '3.0.0'
4
+ VERSION = '3.1.0'
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: safe-pg-migrations
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.0.0
4
+ version: 3.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Matthieu Prat
@@ -11,7 +11,7 @@ authors:
11
11
  autorequire:
12
12
  bindir: bin
13
13
  cert_chain: []
14
- date: 2023-12-11 00:00:00.000000000 Z
14
+ date: 2024-05-02 00:00:00.000000000 Z
15
15
  dependencies:
16
16
  - !ruby/object:Gem::Dependency
17
17
  name: activerecord
@@ -65,6 +65,7 @@ files:
65
65
  - lib/safe-pg-migrations/plugins/statement_insurer.rb
66
66
  - lib/safe-pg-migrations/plugins/statement_insurer/add_column.rb
67
67
  - lib/safe-pg-migrations/plugins/statement_insurer/change_column_null.rb
68
+ - lib/safe-pg-migrations/plugins/statement_insurer/remove_column_index.rb
68
69
  - lib/safe-pg-migrations/plugins/statement_retrier.rb
69
70
  - lib/safe-pg-migrations/plugins/strong_migrations_integration.rb
70
71
  - lib/safe-pg-migrations/plugins/useless_statements_logger.rb
@@ -98,7 +99,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
98
99
  - !ruby/object:Gem::Version
99
100
  version: '0'
100
101
  requirements: []
101
- rubygems_version: 3.3.7
102
+ rubygems_version: 3.4.19
102
103
  signing_key:
103
104
  specification_version: 4
104
105
  summary: Make your PG migrations safe.