safe-pg-migrations 3.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +4 -0
- data/lib/safe-pg-migrations/base.rb +1 -0
- data/lib/safe-pg-migrations/configuration.rb +31 -15
- data/lib/safe-pg-migrations/helpers/batch_over.rb +2 -2
- data/lib/safe-pg-migrations/plugins/statement_insurer/add_column.rb +1 -1
- data/lib/safe-pg-migrations/plugins/statement_insurer/change_column_null.rb +2 -2
- data/lib/safe-pg-migrations/plugins/statement_insurer/remove_column_index.rb +23 -0
- data/lib/safe-pg-migrations/plugins/statement_insurer.rb +4 -2
- data/lib/safe-pg-migrations/plugins/statement_retrier.rb +30 -4
- data/lib/safe-pg-migrations/version.rb +1 -1
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2048cd259a3da82cc4c7456c6c5fa0f9a04752886a864f9a6ed4e891c698c1e5
|
4
|
+
data.tar.gz: bd56fddb533483a264ee89691b7cbd23609a0f6517cc4fdf10f458e9af8152db
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c3ee1fb684f03d3494ab950dd5e7a47be3e36fa710179fa25f2ffba86afb43d671f47d2f9a74f0b286d698181ab93a9d033894ecaba76d1b36976e54b897e788
|
7
|
+
data.tar.gz: 704b4f337c8f9dad758781faf51836889b272a161f08190578ae5d2dcb96fa4b5f6b6de019004f520e32ed9ba136ba9ba31d2c1a34c4df0780ba6a4bf0558d29
|
data/README.md
CHANGED
@@ -329,6 +329,10 @@ SafePgMigrations.config.safe_timeout = 5.seconds # Statement timeout used for al
|
|
329
329
|
|
330
330
|
SafePgMigrations.config.lock_timeout = nil # Lock timeout used for all DDL operations except from CREATE / DROP INDEX. If not set, safe_timeout will be used with a deduction of 1% to ensure that the lock timeout is raised in priority
|
331
331
|
|
332
|
+
SafePgMigrations.config.increase_lock_timeout_on_retry # Activate the lock timeout increase feature on retry if set to true. See max_lock_timeout_for_retry for more information.
|
333
|
+
|
334
|
+
SafePgMigrations.config.max_lock_timeout_for_retry = 1.second # Max lock timeout for the retries for all DDL operations except from CREATE / DROP INDEX. Each retry will increase the lock_timeout (if increase_lock_timeout_on_retry option is set to true) by (max_lock_timeout_for_retry - lock_timeout) / max_tries
|
335
|
+
|
332
336
|
SafePgMigrations.config.blocking_activity_logger_verbose = true # Outputs the raw blocking queries on timeout. When false, outputs information about the lock instead
|
333
337
|
|
334
338
|
SafePgMigrations.config.sensitive_logger = nil # When given, sensitive data will be sent to this logger instead of the standard output. Must implement method `info`.
|
@@ -11,6 +11,7 @@ require 'safe-pg-migrations/plugins/verbose_sql_logger'
|
|
11
11
|
require 'safe-pg-migrations/plugins/blocking_activity_logger'
|
12
12
|
require 'safe-pg-migrations/plugins/statement_insurer/add_column'
|
13
13
|
require 'safe-pg-migrations/plugins/statement_insurer/change_column_null'
|
14
|
+
require 'safe-pg-migrations/plugins/statement_insurer/remove_column_index'
|
14
15
|
require 'safe-pg-migrations/plugins/statement_insurer'
|
15
16
|
require 'safe-pg-migrations/plugins/statement_retrier'
|
16
17
|
require 'safe-pg-migrations/plugins/idempotent_statements'
|
@@ -5,50 +5,66 @@ require 'active_support/core_ext/numeric/time'
|
|
5
5
|
module SafePgMigrations
|
6
6
|
class Configuration
|
7
7
|
attr_accessor(*%i[
|
8
|
+
backfill_batch_size
|
9
|
+
backfill_pause
|
8
10
|
blocking_activity_logger_margin
|
9
11
|
blocking_activity_logger_verbose
|
10
12
|
default_value_backfill_threshold
|
11
|
-
|
12
|
-
backfill_pause
|
13
|
-
retry_delay
|
13
|
+
increase_lock_timeout_on_retry
|
14
14
|
max_tries
|
15
|
+
retry_delay
|
15
16
|
sensitive_logger
|
16
17
|
])
|
17
|
-
attr_reader :lock_timeout, :safe_timeout
|
18
|
+
attr_reader :lock_timeout, :safe_timeout, :max_lock_timeout_for_retry
|
18
19
|
|
19
20
|
def initialize
|
20
|
-
self.default_value_backfill_threshold = nil
|
21
|
-
self.safe_timeout = 5.seconds
|
22
|
-
self.lock_timeout = nil
|
23
|
-
self.blocking_activity_logger_margin = 1.second
|
24
|
-
self.blocking_activity_logger_verbose = true
|
25
21
|
self.backfill_batch_size = 100_000
|
26
22
|
self.backfill_pause = 0.5.second
|
27
|
-
self.
|
23
|
+
self.blocking_activity_logger_margin = 1.second
|
24
|
+
self.blocking_activity_logger_verbose = true
|
25
|
+
self.default_value_backfill_threshold = nil
|
26
|
+
self.increase_lock_timeout_on_retry = false
|
27
|
+
self.lock_timeout = nil
|
28
|
+
self.max_lock_timeout_for_retry = 1.second
|
28
29
|
self.max_tries = 5
|
30
|
+
self.retry_delay = 1.minute
|
31
|
+
self.safe_timeout = 5.seconds
|
29
32
|
self.sensitive_logger = nil
|
30
33
|
end
|
31
34
|
|
32
35
|
def lock_timeout=(value)
|
33
36
|
raise 'Setting lock timeout to 0 disables the lock timeout and is dangerous' if value == 0.seconds
|
34
37
|
|
35
|
-
unless value.nil? || value < safe_timeout
|
36
|
-
raise ArgumentError, "Lock timeout (#{value}) cannot be greater than safe timeout (#{safe_timeout})
|
38
|
+
unless value.nil? || (value < safe_timeout && value <= max_lock_timeout_for_retry)
|
39
|
+
raise ArgumentError, "Lock timeout (#{value}) cannot be greater than the safe timeout (#{safe_timeout}) or the
|
40
|
+
max lock timeout for retry (#{max_lock_timeout_for_retry})"
|
37
41
|
end
|
38
42
|
|
39
43
|
@lock_timeout = value
|
40
44
|
end
|
41
45
|
|
42
46
|
def safe_timeout=(value)
|
43
|
-
|
47
|
+
unless value && value > 0.seconds
|
48
|
+
raise 'Setting safe timeout to 0 or nil disables the safe timeout and is dangerous'
|
49
|
+
end
|
44
50
|
|
45
|
-
unless lock_timeout.nil? || value > lock_timeout
|
46
|
-
raise ArgumentError, "Safe timeout (#{value}) cannot be
|
51
|
+
unless lock_timeout.nil? || (value > lock_timeout && value >= max_lock_timeout_for_retry)
|
52
|
+
raise ArgumentError, "Safe timeout (#{value}) cannot be lower than the lock timeout (#{lock_timeout}) or the
|
53
|
+
max lock timeout for retry (#{max_lock_timeout_for_retry})"
|
47
54
|
end
|
48
55
|
|
49
56
|
@safe_timeout = value
|
50
57
|
end
|
51
58
|
|
59
|
+
def max_lock_timeout_for_retry=(value)
|
60
|
+
unless lock_timeout.nil? || (value >= lock_timeout && value <= safe_timeout)
|
61
|
+
raise ArgumentError, "Max lock timeout for retry (#{value}) cannot be lower than the lock timeout
|
62
|
+
(#{lock_timeout}) and greater than the safe timeout (#{safe_timeout})"
|
63
|
+
end
|
64
|
+
|
65
|
+
@max_lock_timeout_for_retry = value
|
66
|
+
end
|
67
|
+
|
52
68
|
def pg_statement_timeout
|
53
69
|
pg_duration safe_timeout
|
54
70
|
end
|
@@ -19,11 +19,11 @@ module SafePgMigrations
|
|
19
19
|
def next_batch
|
20
20
|
return if endless?
|
21
21
|
|
22
|
-
first = next_scope.take
|
22
|
+
first = next_scope.select(primary_key).take
|
23
23
|
|
24
24
|
return unless first
|
25
25
|
|
26
|
-
last = next_scope.offset(@of).take
|
26
|
+
last = next_scope.select(primary_key).offset(@of).take
|
27
27
|
|
28
28
|
first_key = first[primary_key]
|
29
29
|
last_key = last.nil? ? nil : last[primary_key]
|
@@ -21,7 +21,7 @@ module SafePgMigrations
|
|
21
21
|
null = options.delete(:null)
|
22
22
|
|
23
23
|
Helpers::Logger.say_method_call(:add_column, table_name, column_name, type, options)
|
24
|
-
super
|
24
|
+
super(table_name, column_name, type, **options)
|
25
25
|
|
26
26
|
Helpers::Logger.say_method_call(:change_column_default, table_name, column_name, default)
|
27
27
|
change_column_default(table_name, column_name, default)
|
@@ -16,7 +16,7 @@ module SafePgMigrations
|
|
16
16
|
add_check_constraint table_name, expression, name: constraint_name
|
17
17
|
|
18
18
|
Helpers::Logger.say_method_call :change_column_null, table_name, column_name, false
|
19
|
-
super
|
19
|
+
super(table_name, column_name, false)
|
20
20
|
|
21
21
|
return unless should_remove_constraint? default_name, constraint_name
|
22
22
|
|
@@ -27,7 +27,7 @@ module SafePgMigrations
|
|
27
27
|
private
|
28
28
|
|
29
29
|
def check_constraint_by_expression(table_name, expression)
|
30
|
-
check_constraints(table_name).detect { |check_constraint| check_constraint.expression
|
30
|
+
check_constraints(table_name).detect { |check_constraint| check_constraint.expression == expression }
|
31
31
|
end
|
32
32
|
|
33
33
|
def should_create_constraint?(default, null)
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SafePgMigrations
|
4
|
+
module StatementInsurer
|
5
|
+
module RemoveColumnIndex
|
6
|
+
def remove_column_with_composite_index(table, column)
|
7
|
+
existing_indexes = indexes(table).select { |index|
|
8
|
+
index.columns.size > 1 && index.columns.include?(column.to_s)
|
9
|
+
}
|
10
|
+
|
11
|
+
return unless existing_indexes.any?
|
12
|
+
|
13
|
+
error_message = <<~ERROR
|
14
|
+
Cannot drop column #{column} from table #{table} because composite index(es): #{existing_indexes.map(&:name).join(', ')} is/are present.
|
15
|
+
If they are still required, create the index(es) without #{column} before dropping the existing index(es).
|
16
|
+
Then you will be able to drop the column.
|
17
|
+
ERROR
|
18
|
+
|
19
|
+
raise StandardError, error_message
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -5,6 +5,7 @@ module SafePgMigrations
|
|
5
5
|
include Helpers::SessionSettingManagement
|
6
6
|
include AddColumn
|
7
7
|
include ChangeColumnNull
|
8
|
+
include RemoveColumnIndex
|
8
9
|
|
9
10
|
def validate_check_constraint(table_name, **options)
|
10
11
|
Helpers::Logger.say_method_call :validate_check_constraint, table_name, **options
|
@@ -19,7 +20,7 @@ module SafePgMigrations
|
|
19
20
|
|
20
21
|
Helpers::Logger.say_method_call :add_check_constraint, table_name, expression, **options,
|
21
22
|
validate: false
|
22
|
-
super
|
23
|
+
super(table_name, expression, **options, validate: false)
|
23
24
|
|
24
25
|
return unless options.fetch(:validate, true)
|
25
26
|
|
@@ -42,7 +43,7 @@ module SafePgMigrations
|
|
42
43
|
validate_foreign_key from_table, sub_options.present? ? nil : to_table, **sub_options
|
43
44
|
end
|
44
45
|
|
45
|
-
def create_table(
|
46
|
+
def create_table(table_name, **options)
|
46
47
|
super do |td|
|
47
48
|
yield td if block_given?
|
48
49
|
td.indexes.map! do |key, index_options|
|
@@ -74,6 +75,7 @@ module SafePgMigrations
|
|
74
75
|
foreign_key = foreign_key_for(table_name, column: column_name)
|
75
76
|
|
76
77
|
remove_foreign_key(table_name, name: foreign_key.name) if foreign_key
|
78
|
+
remove_column_with_composite_index(table_name, column_name)
|
77
79
|
super
|
78
80
|
end
|
79
81
|
|
@@ -13,20 +13,46 @@ module SafePgMigrations
|
|
13
13
|
private
|
14
14
|
|
15
15
|
def retry_if_lock_timeout
|
16
|
-
|
16
|
+
initial_lock_timeout = SafePgMigrations.config.lock_timeout
|
17
|
+
number_of_retries = 0
|
17
18
|
begin
|
18
|
-
|
19
|
+
number_of_retries += 1
|
19
20
|
yield
|
20
21
|
rescue ActiveRecord::LockWaitTimeout
|
21
|
-
|
22
|
-
|
22
|
+
# Retrying is useless if we're inside a transaction.
|
23
|
+
if transaction_open? || number_of_retries >= SafePgMigrations.config.max_tries
|
24
|
+
SafePgMigrations.config.lock_timeout = initial_lock_timeout
|
25
|
+
raise
|
26
|
+
end
|
23
27
|
|
24
28
|
retry_delay = SafePgMigrations.config.retry_delay
|
25
29
|
Helpers::Logger.say "Retrying in #{retry_delay} seconds...", sub_item: true
|
30
|
+
|
31
|
+
if SafePgMigrations.config.increase_lock_timeout_on_retry && !SafePgMigrations.config.lock_timeout.nil?
|
32
|
+
increase_lock_timeout
|
33
|
+
end
|
34
|
+
|
26
35
|
sleep retry_delay
|
27
36
|
Helpers::Logger.say 'Retrying now.', sub_item: true
|
28
37
|
retry
|
29
38
|
end
|
30
39
|
end
|
40
|
+
|
41
|
+
def increase_lock_timeout
|
42
|
+
Helpers::Logger.say " Increasing the lock timeout... Currently set to #{SafePgMigrations.config.lock_timeout}",
|
43
|
+
sub_item: true
|
44
|
+
SafePgMigrations.config.lock_timeout = (SafePgMigrations.config.lock_timeout + lock_timeout_step)
|
45
|
+
unless SafePgMigrations.config.lock_timeout < SafePgMigrations.config.max_lock_timeout_for_retry
|
46
|
+
SafePgMigrations.config.lock_timeout = SafePgMigrations.config.max_lock_timeout_for_retry
|
47
|
+
end
|
48
|
+
Helpers::Logger.say " Lock timeout is now set to #{SafePgMigrations.config.lock_timeout}", sub_item: true
|
49
|
+
end
|
50
|
+
|
51
|
+
def lock_timeout_step
|
52
|
+
max_lock_timeout_for_retry = SafePgMigrations.config.max_lock_timeout_for_retry
|
53
|
+
lock_timeout = SafePgMigrations.config.lock_timeout
|
54
|
+
max_tries = SafePgMigrations.config.max_tries
|
55
|
+
@lock_timeout_step ||= (max_lock_timeout_for_retry - lock_timeout) / (max_tries - 1)
|
56
|
+
end
|
31
57
|
end
|
32
58
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: safe-pg-migrations
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Matthieu Prat
|
@@ -11,7 +11,7 @@ authors:
|
|
11
11
|
autorequire:
|
12
12
|
bindir: bin
|
13
13
|
cert_chain: []
|
14
|
-
date:
|
14
|
+
date: 2024-05-02 00:00:00.000000000 Z
|
15
15
|
dependencies:
|
16
16
|
- !ruby/object:Gem::Dependency
|
17
17
|
name: activerecord
|
@@ -65,6 +65,7 @@ files:
|
|
65
65
|
- lib/safe-pg-migrations/plugins/statement_insurer.rb
|
66
66
|
- lib/safe-pg-migrations/plugins/statement_insurer/add_column.rb
|
67
67
|
- lib/safe-pg-migrations/plugins/statement_insurer/change_column_null.rb
|
68
|
+
- lib/safe-pg-migrations/plugins/statement_insurer/remove_column_index.rb
|
68
69
|
- lib/safe-pg-migrations/plugins/statement_retrier.rb
|
69
70
|
- lib/safe-pg-migrations/plugins/strong_migrations_integration.rb
|
70
71
|
- lib/safe-pg-migrations/plugins/useless_statements_logger.rb
|
@@ -98,7 +99,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
98
99
|
- !ruby/object:Gem::Version
|
99
100
|
version: '0'
|
100
101
|
requirements: []
|
101
|
-
rubygems_version: 3.
|
102
|
+
rubygems_version: 3.4.19
|
102
103
|
signing_key:
|
103
104
|
specification_version: 4
|
104
105
|
summary: Make your PG migrations safe.
|