safe-pg-migrations 3.0.1 → 3.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: fe4eda5eb5bed6beac68dd097dec77ab13e1bbe5e3e322bc967b7ebdb2941555
4
- data.tar.gz: 1050659beb7efef8edb36a354dd052e354fb82627e6899f29fe5423118323549
3
+ metadata.gz: 2048cd259a3da82cc4c7456c6c5fa0f9a04752886a864f9a6ed4e891c698c1e5
4
+ data.tar.gz: bd56fddb533483a264ee89691b7cbd23609a0f6517cc4fdf10f458e9af8152db
5
5
  SHA512:
6
- metadata.gz: 904f8d2ef33ce3fb0617d28b980eb1317d6951179b187d76721ec2e8974d326233f78a8efaeb320f683a7d58089f29fc8a484a69cb38d1553b8e2b61f819356f
7
- data.tar.gz: 9720bd9cd4bcf1fa2f300a85abd19c5f12e7ffeef226b8238837c4446a343a23180ab398832891fd9dbcd8210d6e51bb9577e945d11da0c1ef86e48e535902ce
6
+ metadata.gz: c3ee1fb684f03d3494ab950dd5e7a47be3e36fa710179fa25f2ffba86afb43d671f47d2f9a74f0b286d698181ab93a9d033894ecaba76d1b36976e54b897e788
7
+ data.tar.gz: 704b4f337c8f9dad758781faf51836889b272a161f08190578ae5d2dcb96fa4b5f6b6de019004f520e32ed9ba136ba9ba31d2c1a34c4df0780ba6a4bf0558d29
data/README.md CHANGED
@@ -329,6 +329,10 @@ SafePgMigrations.config.safe_timeout = 5.seconds # Statement timeout used for al
329
329
 
330
330
  SafePgMigrations.config.lock_timeout = nil # Lock timeout used for all DDL operations except from CREATE / DROP INDEX. If not set, safe_timeout will be used with a deduction of 1% to ensure that the lock timeout is raised in priority
331
331
 
332
+ SafePgMigrations.config.increase_lock_timeout_on_retry # Activate the lock timeout increase feature on retry if set to true. See max_lock_timeout_for_retry for more information.
333
+
334
+ SafePgMigrations.config.max_lock_timeout_for_retry = 1.second # Max lock timeout for the retries for all DDL operations except from CREATE / DROP INDEX. Each retry will increase the lock_timeout (if increase_lock_timeout_on_retry option is set to true) by (max_lock_timeout_for_retry - lock_timeout) / max_tries
335
+
332
336
  SafePgMigrations.config.blocking_activity_logger_verbose = true # Outputs the raw blocking queries on timeout. When false, outputs information about the lock instead
333
337
 
334
338
  SafePgMigrations.config.sensitive_logger = nil # When given, sensitive data will be sent to this logger instead of the standard output. Must implement method `info`.
@@ -11,6 +11,7 @@ require 'safe-pg-migrations/plugins/verbose_sql_logger'
11
11
  require 'safe-pg-migrations/plugins/blocking_activity_logger'
12
12
  require 'safe-pg-migrations/plugins/statement_insurer/add_column'
13
13
  require 'safe-pg-migrations/plugins/statement_insurer/change_column_null'
14
+ require 'safe-pg-migrations/plugins/statement_insurer/remove_column_index'
14
15
  require 'safe-pg-migrations/plugins/statement_insurer'
15
16
  require 'safe-pg-migrations/plugins/statement_retrier'
16
17
  require 'safe-pg-migrations/plugins/idempotent_statements'
@@ -5,50 +5,66 @@ require 'active_support/core_ext/numeric/time'
5
5
  module SafePgMigrations
6
6
  class Configuration
7
7
  attr_accessor(*%i[
8
+ backfill_batch_size
9
+ backfill_pause
8
10
  blocking_activity_logger_margin
9
11
  blocking_activity_logger_verbose
10
12
  default_value_backfill_threshold
11
- backfill_batch_size
12
- backfill_pause
13
- retry_delay
13
+ increase_lock_timeout_on_retry
14
14
  max_tries
15
+ retry_delay
15
16
  sensitive_logger
16
17
  ])
17
- attr_reader :lock_timeout, :safe_timeout
18
+ attr_reader :lock_timeout, :safe_timeout, :max_lock_timeout_for_retry
18
19
 
19
20
  def initialize
20
- self.default_value_backfill_threshold = nil
21
- self.safe_timeout = 5.seconds
22
- self.lock_timeout = nil
23
- self.blocking_activity_logger_margin = 1.second
24
- self.blocking_activity_logger_verbose = true
25
21
  self.backfill_batch_size = 100_000
26
22
  self.backfill_pause = 0.5.second
27
- self.retry_delay = 1.minute
23
+ self.blocking_activity_logger_margin = 1.second
24
+ self.blocking_activity_logger_verbose = true
25
+ self.default_value_backfill_threshold = nil
26
+ self.increase_lock_timeout_on_retry = false
27
+ self.lock_timeout = nil
28
+ self.max_lock_timeout_for_retry = 1.second
28
29
  self.max_tries = 5
30
+ self.retry_delay = 1.minute
31
+ self.safe_timeout = 5.seconds
29
32
  self.sensitive_logger = nil
30
33
  end
31
34
 
32
35
  def lock_timeout=(value)
33
36
  raise 'Setting lock timeout to 0 disables the lock timeout and is dangerous' if value == 0.seconds
34
37
 
35
- unless value.nil? || value < safe_timeout
36
- raise ArgumentError, "Lock timeout (#{value}) cannot be greater than safe timeout (#{safe_timeout})"
38
+ unless value.nil? || (value < safe_timeout && value <= max_lock_timeout_for_retry)
39
+ raise ArgumentError, "Lock timeout (#{value}) cannot be greater than the safe timeout (#{safe_timeout}) or the
40
+ max lock timeout for retry (#{max_lock_timeout_for_retry})"
37
41
  end
38
42
 
39
43
  @lock_timeout = value
40
44
  end
41
45
 
42
46
  def safe_timeout=(value)
43
- raise 'Setting safe timeout to 0 disables the safe timeout and is dangerous' unless value
47
+ unless value && value > 0.seconds
48
+ raise 'Setting safe timeout to 0 or nil disables the safe timeout and is dangerous'
49
+ end
44
50
 
45
- unless lock_timeout.nil? || value > lock_timeout
46
- raise ArgumentError, "Safe timeout (#{value}) cannot be less than lock timeout (#{lock_timeout})"
51
+ unless lock_timeout.nil? || (value > lock_timeout && value >= max_lock_timeout_for_retry)
52
+ raise ArgumentError, "Safe timeout (#{value}) cannot be lower than the lock timeout (#{lock_timeout}) or the
53
+ max lock timeout for retry (#{max_lock_timeout_for_retry})"
47
54
  end
48
55
 
49
56
  @safe_timeout = value
50
57
  end
51
58
 
59
+ def max_lock_timeout_for_retry=(value)
60
+ unless lock_timeout.nil? || (value >= lock_timeout && value <= safe_timeout)
61
+ raise ArgumentError, "Max lock timeout for retry (#{value}) cannot be lower than the lock timeout
62
+ (#{lock_timeout}) and greater than the safe timeout (#{safe_timeout})"
63
+ end
64
+
65
+ @max_lock_timeout_for_retry = value
66
+ end
67
+
52
68
  def pg_statement_timeout
53
69
  pg_duration safe_timeout
54
70
  end
@@ -19,11 +19,11 @@ module SafePgMigrations
19
19
  def next_batch
20
20
  return if endless?
21
21
 
22
- first = next_scope.take
22
+ first = next_scope.select(primary_key).take
23
23
 
24
24
  return unless first
25
25
 
26
- last = next_scope.offset(@of).take
26
+ last = next_scope.select(primary_key).offset(@of).take
27
27
 
28
28
  first_key = first[primary_key]
29
29
  last_key = last.nil? ? nil : last[primary_key]
@@ -21,7 +21,7 @@ module SafePgMigrations
21
21
  null = options.delete(:null)
22
22
 
23
23
  Helpers::Logger.say_method_call(:add_column, table_name, column_name, type, options)
24
- super table_name, column_name, type, **options
24
+ super(table_name, column_name, type, **options)
25
25
 
26
26
  Helpers::Logger.say_method_call(:change_column_default, table_name, column_name, default)
27
27
  change_column_default(table_name, column_name, default)
@@ -16,7 +16,7 @@ module SafePgMigrations
16
16
  add_check_constraint table_name, expression, name: constraint_name
17
17
 
18
18
  Helpers::Logger.say_method_call :change_column_null, table_name, column_name, false
19
- super table_name, column_name, false
19
+ super(table_name, column_name, false)
20
20
 
21
21
  return unless should_remove_constraint? default_name, constraint_name
22
22
 
@@ -27,7 +27,7 @@ module SafePgMigrations
27
27
  private
28
28
 
29
29
  def check_constraint_by_expression(table_name, expression)
30
- check_constraints(table_name).detect { |check_constraint| check_constraint.expression = expression }
30
+ check_constraints(table_name).detect { |check_constraint| check_constraint.expression == expression }
31
31
  end
32
32
 
33
33
  def should_create_constraint?(default, null)
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SafePgMigrations
4
+ module StatementInsurer
5
+ module RemoveColumnIndex
6
+ def remove_column_with_composite_index(table, column)
7
+ existing_indexes = indexes(table).select { |index|
8
+ index.columns.size > 1 && index.columns.include?(column.to_s)
9
+ }
10
+
11
+ return unless existing_indexes.any?
12
+
13
+ error_message = <<~ERROR
14
+ Cannot drop column #{column} from table #{table} because composite index(es): #{existing_indexes.map(&:name).join(', ')} is/are present.
15
+ If they are still required, create the index(es) without #{column} before dropping the existing index(es).
16
+ Then you will be able to drop the column.
17
+ ERROR
18
+
19
+ raise StandardError, error_message
20
+ end
21
+ end
22
+ end
23
+ end
@@ -5,6 +5,7 @@ module SafePgMigrations
5
5
  include Helpers::SessionSettingManagement
6
6
  include AddColumn
7
7
  include ChangeColumnNull
8
+ include RemoveColumnIndex
8
9
 
9
10
  def validate_check_constraint(table_name, **options)
10
11
  Helpers::Logger.say_method_call :validate_check_constraint, table_name, **options
@@ -19,7 +20,7 @@ module SafePgMigrations
19
20
 
20
21
  Helpers::Logger.say_method_call :add_check_constraint, table_name, expression, **options,
21
22
  validate: false
22
- super table_name, expression, **options, validate: false
23
+ super(table_name, expression, **options, validate: false)
23
24
 
24
25
  return unless options.fetch(:validate, true)
25
26
 
@@ -74,6 +75,7 @@ module SafePgMigrations
74
75
  foreign_key = foreign_key_for(table_name, column: column_name)
75
76
 
76
77
  remove_foreign_key(table_name, name: foreign_key.name) if foreign_key
78
+ remove_column_with_composite_index(table_name, column_name)
77
79
  super
78
80
  end
79
81
 
@@ -13,20 +13,46 @@ module SafePgMigrations
13
13
  private
14
14
 
15
15
  def retry_if_lock_timeout
16
- remaining_tries = SafePgMigrations.config.max_tries
16
+ initial_lock_timeout = SafePgMigrations.config.lock_timeout
17
+ number_of_retries = 0
17
18
  begin
18
- remaining_tries -= 1
19
+ number_of_retries += 1
19
20
  yield
20
21
  rescue ActiveRecord::LockWaitTimeout
21
- raise if transaction_open? # Retrying is useless if we're inside a transaction.
22
- raise unless remaining_tries > 0
22
+ # Retrying is useless if we're inside a transaction.
23
+ if transaction_open? || number_of_retries >= SafePgMigrations.config.max_tries
24
+ SafePgMigrations.config.lock_timeout = initial_lock_timeout
25
+ raise
26
+ end
23
27
 
24
28
  retry_delay = SafePgMigrations.config.retry_delay
25
29
  Helpers::Logger.say "Retrying in #{retry_delay} seconds...", sub_item: true
30
+
31
+ if SafePgMigrations.config.increase_lock_timeout_on_retry && !SafePgMigrations.config.lock_timeout.nil?
32
+ increase_lock_timeout
33
+ end
34
+
26
35
  sleep retry_delay
27
36
  Helpers::Logger.say 'Retrying now.', sub_item: true
28
37
  retry
29
38
  end
30
39
  end
40
+
41
+ def increase_lock_timeout
42
+ Helpers::Logger.say " Increasing the lock timeout... Currently set to #{SafePgMigrations.config.lock_timeout}",
43
+ sub_item: true
44
+ SafePgMigrations.config.lock_timeout = (SafePgMigrations.config.lock_timeout + lock_timeout_step)
45
+ unless SafePgMigrations.config.lock_timeout < SafePgMigrations.config.max_lock_timeout_for_retry
46
+ SafePgMigrations.config.lock_timeout = SafePgMigrations.config.max_lock_timeout_for_retry
47
+ end
48
+ Helpers::Logger.say " Lock timeout is now set to #{SafePgMigrations.config.lock_timeout}", sub_item: true
49
+ end
50
+
51
+ def lock_timeout_step
52
+ max_lock_timeout_for_retry = SafePgMigrations.config.max_lock_timeout_for_retry
53
+ lock_timeout = SafePgMigrations.config.lock_timeout
54
+ max_tries = SafePgMigrations.config.max_tries
55
+ @lock_timeout_step ||= (max_lock_timeout_for_retry - lock_timeout) / (max_tries - 1)
56
+ end
31
57
  end
32
58
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module SafePgMigrations
4
- VERSION = '3.0.1'
4
+ VERSION = '3.1.0'
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: safe-pg-migrations
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.0.1
4
+ version: 3.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Matthieu Prat
@@ -11,7 +11,7 @@ authors:
11
11
  autorequire:
12
12
  bindir: bin
13
13
  cert_chain: []
14
- date: 2023-12-12 00:00:00.000000000 Z
14
+ date: 2024-05-02 00:00:00.000000000 Z
15
15
  dependencies:
16
16
  - !ruby/object:Gem::Dependency
17
17
  name: activerecord
@@ -65,6 +65,7 @@ files:
65
65
  - lib/safe-pg-migrations/plugins/statement_insurer.rb
66
66
  - lib/safe-pg-migrations/plugins/statement_insurer/add_column.rb
67
67
  - lib/safe-pg-migrations/plugins/statement_insurer/change_column_null.rb
68
+ - lib/safe-pg-migrations/plugins/statement_insurer/remove_column_index.rb
68
69
  - lib/safe-pg-migrations/plugins/statement_retrier.rb
69
70
  - lib/safe-pg-migrations/plugins/strong_migrations_integration.rb
70
71
  - lib/safe-pg-migrations/plugins/useless_statements_logger.rb
@@ -98,7 +99,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
98
99
  - !ruby/object:Gem::Version
99
100
  version: '0'
100
101
  requirements: []
101
- rubygems_version: 3.3.7
102
+ rubygems_version: 3.4.19
102
103
  signing_key:
103
104
  specification_version: 4
104
105
  summary: Make your PG migrations safe.