safe-pg-migrations 1.4.2 → 2.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +115 -33
- data/lib/safe-pg-migrations/base.rb +25 -9
- data/lib/safe-pg-migrations/configuration.rb +15 -9
- data/lib/safe-pg-migrations/helpers/batch_over.rb +56 -0
- data/lib/safe-pg-migrations/helpers/blocking_activity_formatter.rb +59 -0
- data/lib/safe-pg-migrations/helpers/blocking_activity_selector.rb +54 -0
- data/lib/safe-pg-migrations/helpers/index_helper.rb +24 -0
- data/lib/safe-pg-migrations/helpers/satisfied_helper.rb +31 -0
- data/lib/safe-pg-migrations/plugins/blocking_activity_logger.rb +51 -80
- data/lib/safe-pg-migrations/plugins/idempotent_statements.rb +53 -13
- data/lib/safe-pg-migrations/plugins/statement_insurer/add_column.rb +52 -0
- data/lib/safe-pg-migrations/plugins/statement_insurer.rb +41 -38
- data/lib/safe-pg-migrations/plugins/statement_retrier.rb +1 -1
- data/lib/safe-pg-migrations/plugins/useless_statements_logger.rb +8 -0
- data/lib/safe-pg-migrations/plugins/verbose_sql_logger.rb +3 -3
- data/lib/safe-pg-migrations/polyfills/index_definition_polyfill.rb +22 -0
- data/lib/safe-pg-migrations/polyfills/verbose_query_logs_polyfill.rb +24 -0
- data/lib/safe-pg-migrations/railtie.rb +1 -0
- data/lib/safe-pg-migrations/version.rb +1 -1
- metadata +17 -27
- data/lib/safe-pg-migrations/plugins/legacy_active_record_support.rb +0 -38
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2fafd98448f0ff697da386a337dc22753be2144feedba4793f87a351581a5607
|
4
|
+
data.tar.gz: 9653ae6395d4c9f27b69dba9dee9142a4535384c42e8d05fac9ca2e2ef1bdce6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 64297c9ba4f040e7afa21fefa2acb2a26e27fb0bfee3d9f9cbf902ee4bb0ac087f4e22591a7b5470cf73dc7bc73345e59904649c02d7540bd4693372e7069549
|
7
|
+
data.tar.gz: 7a2640fd2494480d560cfa6f19059550ecc316eaf729df6d8f97a434c189f4565c49d47acbb02f437d9b1285a466a5615a79a84d6ba1dfb3d94bb54f87b9e788
|
data/README.md
CHANGED
@@ -6,9 +6,9 @@ ActiveRecord migrations for Postgres made safe.
|
|
6
6
|
|
7
7
|
## Requirements
|
8
8
|
|
9
|
-
- Ruby 2.
|
10
|
-
- Rails
|
11
|
-
- PostgreSQL
|
9
|
+
- Ruby 2.7+
|
10
|
+
- Rails 6.0+
|
11
|
+
- PostgreSQL 11.7+
|
12
12
|
|
13
13
|
## Usage
|
14
14
|
|
@@ -25,43 +25,46 @@ gem 'safe-pg-migrations'
|
|
25
25
|
Consider the following migration:
|
26
26
|
|
27
27
|
```rb
|
28
|
-
class
|
28
|
+
class AddPatientRefToAppointments < ActiveRecord::Migration[6.0]
|
29
29
|
def change
|
30
|
-
|
30
|
+
add_reference :appointments, :patient
|
31
31
|
end
|
32
32
|
end
|
33
33
|
```
|
34
34
|
|
35
|
-
If the `users` table is large, running this migration
|
35
|
+
If the `users` table is large, running this migration will likely cause downtime. **Safe PG Migrations** hooks into Active Record so that the following gets executed instead:
|
36
36
|
|
37
37
|
```rb
|
38
|
-
class
|
38
|
+
class AddPatientRefToAppointments < ActiveRecord::Migration[6.0]
|
39
39
|
# Do not wrap the migration in a transaction so that locks are held for a shorter time.
|
40
40
|
disable_ddl_transaction!
|
41
41
|
|
42
42
|
def change
|
43
43
|
# Lower Postgres' lock timeout to avoid statement queueing. Acts like a seatbelt.
|
44
|
-
execute
|
44
|
+
execute("SET lock_timeout TO '5s'")
|
45
45
|
|
46
|
-
#
|
47
|
-
|
46
|
+
# Lower Postgres' statement timeout to avoid too long transactions. Acts like a seatbelt.
|
47
|
+
execute("SET statement_timeout TO '5s'")
|
48
|
+
add_column :appointments, :patient_id, :bigint
|
48
49
|
|
49
|
-
#
|
50
|
-
|
50
|
+
# add_index using the concurrent algorithm, to avoid locking the tables
|
51
|
+
add_index :appointments, :patient_id, algorithm: :concurrently
|
51
52
|
|
52
|
-
#
|
53
|
-
|
53
|
+
# add_foreign_key without validation, to avoid locking the table for too long
|
54
|
+
execute("SET statement_timeout TO '5s'")
|
55
|
+
add_foreign_key :appointments, :patients, validate: false
|
54
56
|
|
55
|
-
|
56
|
-
|
57
|
-
# the
|
58
|
-
|
59
|
-
|
57
|
+
execute("SET statement_timeout TO '0'")
|
58
|
+
|
59
|
+
# validate the foreign key separately, it avoids taking a lock on the entire tables
|
60
|
+
validate_foreign_key :appointments, :patients
|
61
|
+
|
62
|
+
# we also need to set timeouts to their initial values if needed
|
60
63
|
end
|
61
64
|
end
|
62
65
|
```
|
63
66
|
|
64
|
-
Under the hood, **Safe PG Migrations** patches `ActiveRecord::Migration` and extends `ActiveRecord::Base.connection` to make potentially dangerous methods—like `
|
67
|
+
Under the hood, **Safe PG Migrations** patches `ActiveRecord::Migration` and extends `ActiveRecord::Base.connection` to make potentially dangerous methods—like `add_reference`—safe.
|
65
68
|
|
66
69
|
## Motivation
|
67
70
|
|
@@ -106,23 +109,43 @@ Beware though, when adding a volatile default value:
|
|
106
109
|
```ruby
|
107
110
|
add_column :users, :created_at, default: 'clock_timestamp()'
|
108
111
|
```
|
109
|
-
PG will still needs to update every row of the table, and will most likely statement timeout for big table. In this case,
|
112
|
+
PG will still needs to update every row of the table, and will most likely statement timeout for big table. In this case, **Safe PG Migrations** can automatically backfill data when the option `default_value_backfill:` is set to `:update_in_batches`.
|
113
|
+
|
114
|
+
</details>
|
115
|
+
|
116
|
+
<details>
|
117
|
+
<summary>Safe add_column - adding a volatile default value</summary>
|
118
|
+
|
119
|
+
**Safe PG Migrations** provides the extra option parameter `default_value_backfill:`. When your migration is adding a volatile default value, the option `:update_in_batches` can be set. It will automatically backfill the value in a safe manner.
|
120
|
+
|
121
|
+
```ruby
|
122
|
+
add_column :users, :created_at, default: 'clock_timestamp()', default_value_backfill: :update_in_batches
|
123
|
+
```
|
124
|
+
|
125
|
+
More specifically, it will:
|
126
|
+
|
127
|
+
1. create the column without default value and without null constraint. This ensure the `ACCESS EXCLUSIVE` lock is acquired for the least amount of time;
|
128
|
+
2. add the default value, without data backfill. An `ACCESS EXCLUSIVE` lock is acquired and released immediately;
|
129
|
+
3. backfill data, in batch of `SafePgMigrations.config.backfill_batch_size` and with a pause of `SafePgMigrations.config.backfill_pause` between each batch;
|
130
|
+
4. change the column to `null: false`, if defined in the parameters, following the algorithm we have defined below.
|
110
131
|
|
111
|
-
|
132
|
+
---
|
133
|
+
**NOTE**
|
112
134
|
|
113
|
-
|
114
|
-
Adding a column with a default value and a not-null constraint is [dangerous](https://wework.github.io/data/2015/11/05/add-columns-with-default-values-to-large-tables-in-rails-postgres/).
|
135
|
+
Data backfill take time. If your table is big, your migrations will (safely) hangs for a while. You might want to backfill data manually instead, to do so you will need two migrations
|
115
136
|
|
116
|
-
|
137
|
+
1. First migration :
|
117
138
|
|
118
|
-
|
119
|
-
2. Then set the default value on the column,
|
120
|
-
3. Then backfilling the column,
|
121
|
-
4. And then adding the not null constraint with a short statement timeout.
|
139
|
+
a. adds the column without default and without null constraint;
|
122
140
|
|
123
|
-
|
141
|
+
b. add the default value.
|
124
142
|
|
125
|
-
|
143
|
+
2. manual data backfill (rake task, manual operation, ...)
|
144
|
+
3. Second migration which change the column to null false (with **Safe PG Migrations**, `change_column_null` is safe and can be used; see section below)
|
145
|
+
|
146
|
+
---
|
147
|
+
|
148
|
+
`default_value_backfill:` also accept the value `:auto` which is set by default. In this case, **Safe PG Migrations** will not backfill data and will let PostgreSQL handle it itself.
|
126
149
|
|
127
150
|
</details>
|
128
151
|
|
@@ -143,13 +166,70 @@ If you still get lock timeout while adding / removing indexes, it might be for o
|
|
143
166
|
|
144
167
|
Adding a foreign key requires a `SHARE ROW EXCLUSIVE` lock, which **prevent writing in the tables** while the migration is running.
|
145
168
|
|
146
|
-
Adding the constraint itself is rather fast, the major part of the time is spent on validating this constraint. Thus
|
169
|
+
Adding the constraint itself is rather fast, the major part of the time is spent on validating this constraint. Thus **Safe PG Migrations** ensures that adding a foreign key holds blocking locks for the least amount of time by splitting the foreign key creation in two steps:
|
147
170
|
|
148
171
|
1. adding the constraint *without validation*, will not validate existing rows;
|
149
172
|
2. validating the constraint, will validate existing rows in the table, without blocking read or write on the table
|
150
173
|
|
151
174
|
</details>
|
152
175
|
|
176
|
+
|
177
|
+
<details><summary id="safe_add_check_constraint">Safe <code>add_check_constraint</code> (ActiveRecord > 6.1)</summary>
|
178
|
+
|
179
|
+
Adding a check constraint requires an `ACCESS EXCLUSIVE` lock, which **prevent writing and reading in the tables** [as soon as the lock is requested](https://medium.com/doctolib/stop-worrying-about-postgresql-locks-in-your-rails-migrations-3426027e9cc9).
|
180
|
+
|
181
|
+
Adding the constraint itself is rather fast, the major part of the time is spent on validating this constraint.
|
182
|
+
Thus **Safe PG Migrations** ensures that adding a constraints holds blocking locks for the least amount of time by
|
183
|
+
splitting the constraint addition in two steps:
|
184
|
+
|
185
|
+
1. adding the constraint *without validation*, will not validate existing rows;
|
186
|
+
2. validating the constraint, will validate existing rows in the table, without blocking read or write on the table
|
187
|
+
|
188
|
+
</details>
|
189
|
+
|
190
|
+
<details><summary id="safe_change_column_null">Safe <code>change_column_null</code> (ActiveRecord and PG version dependant)</summary>
|
191
|
+
|
192
|
+
Changing the nullability of a column requires an `ACCESS EXCLUSIVE` lock, which **prevent writing and reading in the tables** [as soon as the lock is requested](https://medium.com/doctolib/stop-worrying-about-postgresql-locks-in-your-rails-migrations-3426027e9cc9).
|
193
|
+
|
194
|
+
Adding the constraint itself is rather fast, the major part of the time is spent on validating this constraint.
|
195
|
+
|
196
|
+
**Safe PG Migrations** acts differently depending on the version you are on.
|
197
|
+
|
198
|
+
### Recent versions of PG and Active Record (> 12 and > 6.1)
|
199
|
+
|
200
|
+
Starting on PostgreSQL versions 12, adding the column NOT NULL constraint is safe if a check constraint validates the
|
201
|
+
nullability of the same column. **Safe PG Migrations** also relies on add_check_constraint, which was introduced in
|
202
|
+
ActiveRecord 6.1.
|
203
|
+
|
204
|
+
If these requirements are met, **Safe PG Migrations** ensures that adding a constraints holds blocking locks for the least
|
205
|
+
amount of time by splitting the constraint addition in several steps:
|
206
|
+
|
207
|
+
1. adding a `IS NOT NULL` constraint *without validation*, will not validate existing rows but block read or write;
|
208
|
+
2. validating the constraint, will validate existing rows in the table, without blocking read or write on the table;
|
209
|
+
3. changing the not null status of the column, thanks to the NOT NULL constraint without having to scan the table sequentially;
|
210
|
+
4. dropping the `IS NOT NULL` constraint.
|
211
|
+
|
212
|
+
### Older versions of PG or ActiveRecord
|
213
|
+
|
214
|
+
If the version of PostgreSQL is below 12, or if the version of ActiveRecord is below 6.1, **Safe PG Migrations** will only
|
215
|
+
wrap ActiveRecord method into a statement timeout and lock timeout.
|
216
|
+
|
217
|
+
### Call with a default parameter
|
218
|
+
|
219
|
+
Calling change_column_null with a default parameter [is dangerous](https://github.com/rails/rails/blob/716baea69f989b64f5bfeaff880c2512377bebab/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb#L446)
|
220
|
+
and is likely not to finish in the statement timeout defined by **Safe PG Migrations**. For this reason, when the default
|
221
|
+
parameter is given, **Safe PG Migrations** will simply forward it to activerecord methods without trying to improve it
|
222
|
+
|
223
|
+
### Dropping a NULL constraint
|
224
|
+
|
225
|
+
Dropping a null constraint still requires an `ACCESS EXCLUSIVE` lock, but does not require extra operation to reduce the
|
226
|
+
amount of time during which the lock is held. **Safe PG Migrations** only wrap methods of activerecord in lock and statement
|
227
|
+
timeouts
|
228
|
+
|
229
|
+
</details>
|
230
|
+
|
231
|
+
|
232
|
+
|
153
233
|
<details><summary>Retry after lock timeout</summary>
|
154
234
|
|
155
235
|
When a statement fails with a lock timeout, **Safe PG Migrations** retries it (5 times max) [list of retriable statements](https://github.com/doctolib/safe-pg-migrations/blob/66933256252b6bbf12e404b829a361dbba30e684/lib/safe-pg-migrations/plugins/statement_retrier.rb#L5)
|
@@ -214,7 +294,9 @@ SafePgMigrations.config.blocking_activity_logger_verbose = true # Outputs the ra
|
|
214
294
|
|
215
295
|
SafePgMigrations.config.blocking_activity_logger_margin = 1.second # Delay to output blocking queries before timeout. Must be shorter than safe_timeout
|
216
296
|
|
217
|
-
SafePgMigrations.config.
|
297
|
+
SafePgMigrations.config.backfill_batch_size = 100_000 # Size of the batches used for backfilling when adding a column with a default value
|
298
|
+
|
299
|
+
SafePgMigrations.config.backfill_pause = 0.5.second # Delay between each batch during a backfill. This ensure replication can happen safely.
|
218
300
|
|
219
301
|
SafePgMigrations.config.retry_delay = 1.minute # Delay between retries for retryable statements
|
220
302
|
|
@@ -1,14 +1,18 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require 'ruby2_keywords'
|
4
3
|
require 'safe-pg-migrations/configuration'
|
4
|
+
require 'safe-pg-migrations/helpers/satisfied_helper'
|
5
|
+
require 'safe-pg-migrations/helpers/index_helper'
|
6
|
+
require 'safe-pg-migrations/helpers/batch_over'
|
5
7
|
require 'safe-pg-migrations/plugins/verbose_sql_logger'
|
6
8
|
require 'safe-pg-migrations/plugins/blocking_activity_logger'
|
9
|
+
require 'safe-pg-migrations/plugins/statement_insurer/add_column'
|
7
10
|
require 'safe-pg-migrations/plugins/statement_insurer'
|
8
11
|
require 'safe-pg-migrations/plugins/statement_retrier'
|
9
12
|
require 'safe-pg-migrations/plugins/idempotent_statements'
|
10
13
|
require 'safe-pg-migrations/plugins/useless_statements_logger'
|
11
|
-
require 'safe-pg-migrations/
|
14
|
+
require 'safe-pg-migrations/polyfills/index_definition_polyfill'
|
15
|
+
require 'safe-pg-migrations/polyfills/verbose_query_logs_polyfill'
|
12
16
|
|
13
17
|
module SafePgMigrations
|
14
18
|
# Order matters: the bottom-most plugin will have precedence
|
@@ -18,30 +22,26 @@ module SafePgMigrations
|
|
18
22
|
StatementRetrier,
|
19
23
|
StatementInsurer,
|
20
24
|
UselessStatementsLogger,
|
21
|
-
|
25
|
+
Polyfills::IndexDefinitionPolyfill,
|
22
26
|
].freeze
|
23
27
|
|
24
28
|
class << self
|
25
29
|
attr_reader :current_migration, :pg_version_num
|
26
30
|
|
27
|
-
def setup_and_teardown(migration, connection)
|
31
|
+
def setup_and_teardown(migration, connection, &block)
|
28
32
|
@pg_version_num = get_pg_version_num(connection)
|
29
33
|
@alternate_connection = nil
|
30
34
|
@current_migration = migration
|
31
35
|
stdout_sql_logger = VerboseSqlLogger.new.setup if verbose?
|
32
36
|
PLUGINS.each { |plugin| connection.extend(plugin) }
|
33
37
|
|
34
|
-
connection.with_setting
|
38
|
+
connection.with_setting :lock_timeout, SafePgMigrations.config.pg_lock_timeout, &block
|
35
39
|
ensure
|
36
40
|
close_alternate_connection
|
37
41
|
@current_migration = nil
|
38
42
|
stdout_sql_logger&.teardown
|
39
43
|
end
|
40
44
|
|
41
|
-
def get_pg_version_num(connection)
|
42
|
-
connection.query_value('SHOW server_version_num').to_i
|
43
|
-
end
|
44
|
-
|
45
45
|
def alternate_connection
|
46
46
|
@alternate_connection ||= ActiveRecord::Base.connection_pool.send(:new_connection)
|
47
47
|
end
|
@@ -64,6 +64,9 @@ module SafePgMigrations
|
|
64
64
|
end
|
65
65
|
|
66
66
|
def verbose?
|
67
|
+
unless current_migration.class._safe_pg_migrations_verbose.nil?
|
68
|
+
return current_migration.class._safe_pg_migrations_verbose
|
69
|
+
end
|
67
70
|
return ENV['SAFE_PG_MIGRATIONS_VERBOSE'] == '1' if ENV['SAFE_PG_MIGRATIONS_VERBOSE']
|
68
71
|
return Rails.env.production? if defined?(Rails)
|
69
72
|
|
@@ -73,9 +76,21 @@ module SafePgMigrations
|
|
73
76
|
def config
|
74
77
|
@config ||= Configuration.new
|
75
78
|
end
|
79
|
+
|
80
|
+
def get_pg_version_num(connection)
|
81
|
+
connection.query_value('SHOW server_version_num').to_i
|
82
|
+
end
|
76
83
|
end
|
77
84
|
|
78
85
|
module Migration
|
86
|
+
module ClassMethods
|
87
|
+
attr_accessor :_safe_pg_migrations_verbose
|
88
|
+
|
89
|
+
def safe_pg_migrations_verbose(verbose)
|
90
|
+
@_safe_pg_migrations_verbose = verbose
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
79
94
|
def exec_migration(connection, direction)
|
80
95
|
SafePgMigrations.setup_and_teardown(self, connection) do
|
81
96
|
super(connection, direction)
|
@@ -95,6 +110,7 @@ module SafePgMigrations
|
|
95
110
|
add_belongs_to
|
96
111
|
change_column_null
|
97
112
|
add_foreign_key
|
113
|
+
add_check_constraint
|
98
114
|
].freeze
|
99
115
|
|
100
116
|
SAFE_METHODS.each do |method|
|
@@ -4,26 +4,32 @@ require 'active_support/core_ext/numeric/time'
|
|
4
4
|
|
5
5
|
module SafePgMigrations
|
6
6
|
class Configuration
|
7
|
-
attr_accessor :safe_timeout
|
8
|
-
|
9
|
-
attr_accessor :blocking_activity_logger_verbose
|
10
|
-
attr_accessor :batch_size
|
11
|
-
attr_accessor :retry_delay
|
12
|
-
attr_accessor :max_tries
|
7
|
+
attr_accessor :safe_timeout, :blocking_activity_logger_margin, :blocking_activity_logger_verbose,
|
8
|
+
:backfill_batch_size, :backfill_pause, :retry_delay, :max_tries
|
13
9
|
|
14
10
|
def initialize
|
15
11
|
self.safe_timeout = 5.seconds
|
16
12
|
self.blocking_activity_logger_margin = 1.second
|
17
13
|
self.blocking_activity_logger_verbose = true
|
18
|
-
self.
|
14
|
+
self.backfill_batch_size = 100_000
|
15
|
+
self.backfill_pause = 0.5.second
|
19
16
|
self.retry_delay = 1.minute
|
20
17
|
self.max_tries = 5
|
21
18
|
end
|
22
19
|
|
23
|
-
def
|
24
|
-
pg_duration
|
20
|
+
def pg_statement_timeout
|
21
|
+
pg_duration safe_timeout
|
25
22
|
end
|
26
23
|
|
24
|
+
def pg_lock_timeout
|
25
|
+
# if statement timeout and lock timeout have the same value, statement timeout will raise in priority. We actually
|
26
|
+
# need the opposite for BlockingActivityLogger to detect lock timeouts correctly.
|
27
|
+
# By reducing the lock timeout by a very small margin, we ensure that the lock timeout is raised in priority
|
28
|
+
pg_duration safe_timeout * 0.99
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
27
33
|
def pg_duration(duration)
|
28
34
|
value, unit = duration.integer? ? [duration, 's'] : [(duration * 1000).to_i, 'ms']
|
29
35
|
"#{value}#{unit}"
|
@@ -0,0 +1,56 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SafePgMigrations
|
4
|
+
module Helpers
|
5
|
+
class BatchOver
|
6
|
+
def initialize(model, of: SafePgMigrations.config.backfill_batch_size)
|
7
|
+
@model = model
|
8
|
+
@of = of
|
9
|
+
|
10
|
+
@current_range = nil
|
11
|
+
end
|
12
|
+
|
13
|
+
def each_batch
|
14
|
+
yield scope.where(primary_key => @current_range) while next_batch
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
def next_batch
|
20
|
+
return if endless?
|
21
|
+
|
22
|
+
first = next_scope.take
|
23
|
+
|
24
|
+
return unless first
|
25
|
+
|
26
|
+
last = next_scope.offset(@of).take
|
27
|
+
|
28
|
+
first_key = first[primary_key]
|
29
|
+
last_key = last.nil? ? nil : last[primary_key]
|
30
|
+
|
31
|
+
@current_range = first_key...last_key
|
32
|
+
end
|
33
|
+
|
34
|
+
def next_scope
|
35
|
+
return scope if @current_range.nil?
|
36
|
+
return scope.none if endless?
|
37
|
+
|
38
|
+
scope.where(primary_key => @current_range.end..)
|
39
|
+
end
|
40
|
+
|
41
|
+
def scope
|
42
|
+
@model.order(primary_key => :asc)
|
43
|
+
end
|
44
|
+
|
45
|
+
def endless?
|
46
|
+
return false if @current_range.nil?
|
47
|
+
|
48
|
+
@current_range.end.nil?
|
49
|
+
end
|
50
|
+
|
51
|
+
def primary_key
|
52
|
+
@model.primary_key
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SafePgMigrations
|
4
|
+
module Helpers
|
5
|
+
module BlockingActivityFormatter
|
6
|
+
def log_queries(queries)
|
7
|
+
if queries.empty?
|
8
|
+
SafePgMigrations.say 'Could not find any blocking query.', true
|
9
|
+
else
|
10
|
+
SafePgMigrations.say(
|
11
|
+
"Statement was being blocked by the following #{'query'.pluralize(queries.size)}:",
|
12
|
+
true
|
13
|
+
)
|
14
|
+
SafePgMigrations.say '', true
|
15
|
+
output_blocking_queries(queries)
|
16
|
+
SafePgMigrations.say(
|
17
|
+
'Beware, some of those queries might run in a transaction. In this case the locking query might be ' \
|
18
|
+
'located elsewhere in the transaction',
|
19
|
+
true
|
20
|
+
)
|
21
|
+
SafePgMigrations.say '', true
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
def output_blocking_queries(queries)
|
28
|
+
if SafePgMigrations.config.blocking_activity_logger_verbose
|
29
|
+
queries.each do |pid, query, start_time|
|
30
|
+
SafePgMigrations.say "Query with pid #{pid || 'null'} started #{format_start_time start_time}: #{query}",
|
31
|
+
true
|
32
|
+
end
|
33
|
+
else
|
34
|
+
output_confidentially_blocking_queries(queries)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def output_confidentially_blocking_queries(queries)
|
39
|
+
queries.each do |start_time, locktype, mode, pid, transactionid|
|
40
|
+
SafePgMigrations.say(
|
41
|
+
"Query with pid #{pid || 'null'} " \
|
42
|
+
"started #{format_start_time(start_time)}: " \
|
43
|
+
"lock type: #{locktype || 'null'}, " \
|
44
|
+
"lock mode: #{mode || 'null'}, " \
|
45
|
+
"lock transactionid: #{transactionid || 'null'}",
|
46
|
+
true
|
47
|
+
)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def format_start_time(start_time, reference_time = Time.now)
|
52
|
+
start_time = Time.parse(start_time) unless start_time.is_a? Time
|
53
|
+
|
54
|
+
duration = (reference_time - start_time).round
|
55
|
+
"#{duration} #{'second'.pluralize(duration)} ago"
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SafePgMigrations
|
4
|
+
module Helpers
|
5
|
+
module BlockingActivitySelector
|
6
|
+
FILTERED_COLUMNS = %w[
|
7
|
+
blocked_activity.xact_start
|
8
|
+
blocked_locks.locktype
|
9
|
+
blocked_locks.mode
|
10
|
+
blocking_activity.pid
|
11
|
+
blocked_locks.transactionid
|
12
|
+
].freeze
|
13
|
+
|
14
|
+
VERBOSE_COLUMNS = %w[
|
15
|
+
blocking_activity.pid
|
16
|
+
blocking_activity.query
|
17
|
+
blocked_activity.xact_start
|
18
|
+
].freeze
|
19
|
+
|
20
|
+
def select_blocking_queries_sql
|
21
|
+
columns =
|
22
|
+
(
|
23
|
+
if SafePgMigrations.config.blocking_activity_logger_verbose
|
24
|
+
VERBOSE_COLUMNS
|
25
|
+
else
|
26
|
+
FILTERED_COLUMNS
|
27
|
+
end
|
28
|
+
)
|
29
|
+
|
30
|
+
<<~SQL.squish
|
31
|
+
SELECT #{columns.join(', ')}
|
32
|
+
FROM pg_catalog.pg_locks blocked_locks
|
33
|
+
JOIN pg_catalog.pg_stat_activity blocked_activity
|
34
|
+
ON blocked_activity.pid = blocked_locks.pid
|
35
|
+
JOIN pg_catalog.pg_locks blocking_locks
|
36
|
+
ON blocking_locks.locktype = blocked_locks.locktype
|
37
|
+
AND blocking_locks.DATABASE IS NOT DISTINCT FROM blocked_locks.DATABASE
|
38
|
+
AND blocking_locks.relation IS NOT DISTINCT FROM blocked_locks.relation
|
39
|
+
AND blocking_locks.page IS NOT DISTINCT FROM blocked_locks.page
|
40
|
+
AND blocking_locks.tuple IS NOT DISTINCT FROM blocked_locks.tuple
|
41
|
+
AND blocking_locks.virtualxid IS NOT DISTINCT FROM blocked_locks.virtualxid
|
42
|
+
AND blocking_locks.transactionid IS NOT DISTINCT FROM blocked_locks.transactionid
|
43
|
+
AND blocking_locks.classid IS NOT DISTINCT FROM blocked_locks.classid
|
44
|
+
AND blocking_locks.objid IS NOT DISTINCT FROM blocked_locks.objid
|
45
|
+
AND blocking_locks.objsubid IS NOT DISTINCT FROM blocked_locks.objsubid
|
46
|
+
AND blocking_locks.pid != blocked_locks.pid
|
47
|
+
JOIN pg_catalog.pg_stat_activity blocking_activity
|
48
|
+
ON blocking_activity.pid = blocking_locks.pid
|
49
|
+
WHERE blocked_locks.pid = %d
|
50
|
+
SQL
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SafePgMigrations
|
4
|
+
module Helpers
|
5
|
+
module IndexHelper
|
6
|
+
def index_definition(table_name, column_name, **options)
|
7
|
+
index_definition, = add_index_options(table_name, column_name, **options)
|
8
|
+
index_definition
|
9
|
+
end
|
10
|
+
|
11
|
+
private
|
12
|
+
|
13
|
+
def index_valid?(index_name)
|
14
|
+
query_value <<~SQL.squish
|
15
|
+
SELECT indisvalid
|
16
|
+
FROM pg_index i
|
17
|
+
JOIN pg_class c
|
18
|
+
ON i.indexrelid = c.oid
|
19
|
+
WHERE c.relname = '#{index_name}';
|
20
|
+
SQL
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SafePgMigrations
|
4
|
+
module Helpers
|
5
|
+
module SatisfiedHelper
|
6
|
+
class << self
|
7
|
+
def satisfies_change_column_null_requirements?
|
8
|
+
satisfies_add_check_constraints? && SafePgMigrations.pg_version_num >= 120_000
|
9
|
+
end
|
10
|
+
|
11
|
+
def satisfies_add_check_constraints!
|
12
|
+
return if satisfies_add_check_constraints?
|
13
|
+
|
14
|
+
raise NotImplementedError, 'add_check_constraint is not supported in your ActiveRecord version'
|
15
|
+
end
|
16
|
+
|
17
|
+
def satisfies_add_check_constraints?
|
18
|
+
satisfied? '>=6.1.0'
|
19
|
+
end
|
20
|
+
|
21
|
+
def satisfies_add_column_update_rows_backfill?
|
22
|
+
satisfies_change_column_null_requirements?
|
23
|
+
end
|
24
|
+
|
25
|
+
def satisfied?(version)
|
26
|
+
Gem::Requirement.new(version).satisfied_by? Gem::Version.new(::ActiveRecord::VERSION::STRING)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -1,65 +1,62 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require_relative '../helpers/blocking_activity_formatter'
|
4
|
+
require_relative '../helpers/blocking_activity_selector'
|
5
|
+
|
3
6
|
module SafePgMigrations
|
4
|
-
module BlockingActivityLogger
|
5
|
-
|
6
|
-
|
7
|
-
blocked_locks.locktype
|
8
|
-
blocked_locks.mode
|
9
|
-
blocking_activity.pid
|
10
|
-
blocked_locks.transactionid
|
11
|
-
].freeze
|
12
|
-
|
13
|
-
VERBOSE_COLUMNS = %w[
|
14
|
-
blocking_activity.query
|
15
|
-
blocked_activity.xact_start
|
16
|
-
].freeze
|
7
|
+
module BlockingActivityLogger
|
8
|
+
include ::SafePgMigrations::Helpers::BlockingActivityFormatter
|
9
|
+
include ::SafePgMigrations::Helpers::BlockingActivitySelector
|
17
10
|
|
18
11
|
%i[
|
19
|
-
add_column
|
12
|
+
add_column
|
13
|
+
remove_column
|
14
|
+
add_foreign_key
|
15
|
+
remove_foreign_key
|
16
|
+
change_column_default
|
17
|
+
change_column_null
|
18
|
+
create_table
|
20
19
|
].each do |method|
|
21
20
|
define_method method do |*args, &block|
|
22
|
-
|
21
|
+
log_blocking_queries_after_lock { super(*args, &block) }
|
23
22
|
end
|
24
23
|
ruby2_keywords method
|
25
24
|
end
|
26
25
|
|
27
|
-
|
26
|
+
%i[add_index remove_index].each do |method|
|
27
|
+
define_method method do |*args, **options, &block|
|
28
|
+
return super(*args, **options, &block) if options[:algorithm] != :concurrently
|
28
29
|
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
<<~SQL.squish
|
33
|
-
SELECT #{columns.join(', ')}
|
34
|
-
FROM pg_catalog.pg_locks blocked_locks
|
35
|
-
JOIN pg_catalog.pg_stat_activity blocked_activity
|
36
|
-
ON blocked_activity.pid = blocked_locks.pid
|
37
|
-
JOIN pg_catalog.pg_locks blocking_locks
|
38
|
-
ON blocking_locks.locktype = blocked_locks.locktype
|
39
|
-
AND blocking_locks.DATABASE IS NOT DISTINCT FROM blocked_locks.DATABASE
|
40
|
-
AND blocking_locks.relation IS NOT DISTINCT FROM blocked_locks.relation
|
41
|
-
AND blocking_locks.page IS NOT DISTINCT FROM blocked_locks.page
|
42
|
-
AND blocking_locks.tuple IS NOT DISTINCT FROM blocked_locks.tuple
|
43
|
-
AND blocking_locks.virtualxid IS NOT DISTINCT FROM blocked_locks.virtualxid
|
44
|
-
AND blocking_locks.transactionid IS NOT DISTINCT FROM blocked_locks.transactionid
|
45
|
-
AND blocking_locks.classid IS NOT DISTINCT FROM blocked_locks.classid
|
46
|
-
AND blocking_locks.objid IS NOT DISTINCT FROM blocked_locks.objid
|
47
|
-
AND blocking_locks.objsubid IS NOT DISTINCT FROM blocked_locks.objsubid
|
48
|
-
AND blocking_locks.pid != blocked_locks.pid
|
49
|
-
JOIN pg_catalog.pg_stat_activity blocking_activity
|
50
|
-
ON blocking_activity.pid = blocking_locks.pid
|
51
|
-
WHERE blocked_locks.pid = %d
|
52
|
-
SQL
|
30
|
+
log_blocking_queries_loop { super(*args, **options, &block) }
|
31
|
+
end
|
53
32
|
end
|
54
33
|
|
55
|
-
|
56
|
-
|
57
|
-
|
34
|
+
private
|
35
|
+
|
36
|
+
def log_blocking_queries_loop
|
37
|
+
blocking_queries_retriever_thread =
|
38
|
+
Thread.new do
|
39
|
+
loop do
|
40
|
+
sleep SafePgMigrations.config.retry_delay
|
41
|
+
|
42
|
+
log_queries SafePgMigrations.alternate_connection.query(
|
43
|
+
select_blocking_queries_sql % raw_connection.backend_pid
|
44
|
+
)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
yield
|
49
|
+
|
50
|
+
blocking_queries_retriever_thread.kill
|
51
|
+
end
|
58
52
|
|
53
|
+
def log_blocking_queries_after_lock
|
59
54
|
blocking_queries_retriever_thread =
|
60
55
|
Thread.new do
|
61
56
|
sleep delay_before_logging
|
62
|
-
SafePgMigrations.alternate_connection.query(
|
57
|
+
SafePgMigrations.alternate_connection.query(
|
58
|
+
select_blocking_queries_sql % raw_connection.backend_pid
|
59
|
+
)
|
63
60
|
end
|
64
61
|
|
65
62
|
yield
|
@@ -71,51 +68,25 @@ module SafePgMigrations
|
|
71
68
|
begin
|
72
69
|
blocking_queries_retriever_thread.value
|
73
70
|
rescue StandardError => e
|
74
|
-
SafePgMigrations.say(
|
71
|
+
SafePgMigrations.say(
|
72
|
+
"Error while retrieving blocking queries: #{e}",
|
73
|
+
true
|
74
|
+
)
|
75
75
|
nil
|
76
76
|
end
|
77
77
|
|
78
|
-
|
79
|
-
|
80
|
-
if queries.empty?
|
81
|
-
SafePgMigrations.say 'Could not find any blocking query.', true
|
82
|
-
else
|
83
|
-
SafePgMigrations.say(
|
84
|
-
"Statement was being blocked by the following #{'query'.pluralize(queries.size)}:", true
|
85
|
-
)
|
86
|
-
SafePgMigrations.say '', true
|
87
|
-
output_blocking_queries(queries)
|
88
|
-
SafePgMigrations.say(
|
89
|
-
'Beware, some of those queries might run in a transaction. In this case the locking query might be '\
|
90
|
-
'located elsewhere in the transaction',
|
91
|
-
true
|
92
|
-
)
|
93
|
-
SafePgMigrations.say '', true
|
94
|
-
end
|
78
|
+
log_queries queries unless queries.nil?
|
95
79
|
|
96
80
|
raise
|
97
81
|
end
|
98
82
|
|
99
|
-
def
|
100
|
-
|
101
|
-
|
102
|
-
else
|
103
|
-
queries.each do |start_time, locktype, mode, pid, transactionid|
|
104
|
-
SafePgMigrations.say(
|
105
|
-
"#{format_start_time(start_time)}: lock type: #{locktype || 'null'}, " \
|
106
|
-
"lock mode: #{mode || 'null'}, " \
|
107
|
-
"lock pid: #{pid || 'null'}, " \
|
108
|
-
"lock transactionid: #{transactionid || 'null'}",
|
109
|
-
true
|
110
|
-
)
|
111
|
-
end
|
112
|
-
end
|
83
|
+
def delay_before_logging
|
84
|
+
SafePgMigrations.config.safe_timeout -
|
85
|
+
SafePgMigrations.config.blocking_activity_logger_margin
|
113
86
|
end
|
114
87
|
|
115
|
-
def
|
116
|
-
|
117
|
-
duration = (reference_time - start_time).round
|
118
|
-
"transaction started #{duration} #{'second'.pluralize(duration)} ago"
|
88
|
+
def delay_before_retry
|
89
|
+
SafePgMigrations.config.blocking_activity_logger_margin + SafePgMigrations.config.retry_delay
|
119
90
|
end
|
120
91
|
end
|
121
92
|
end
|
@@ -2,6 +2,8 @@
|
|
2
2
|
|
3
3
|
module SafePgMigrations
|
4
4
|
module IdempotentStatements
|
5
|
+
include SafePgMigrations::Helpers::IndexHelper
|
6
|
+
|
5
7
|
ruby2_keywords def add_index(table_name, column_name, *args)
|
6
8
|
options = args.last.is_a?(Hash) ? args.last : {}
|
7
9
|
|
@@ -53,6 +55,16 @@ module SafePgMigrations
|
|
53
55
|
)
|
54
56
|
end
|
55
57
|
|
58
|
+
def remove_foreign_key(from_table, to_table = nil, **options)
|
59
|
+
return super if foreign_key_exists?(from_table, to_table, **options)
|
60
|
+
|
61
|
+
reference_name = to_table || options[:to_table] || options[:column] || options[:name]
|
62
|
+
SafePgMigrations.say(
|
63
|
+
"/!\\ Foreign key '#{from_table}' -> '#{reference_name}' does not exist. Skipping statement.",
|
64
|
+
true
|
65
|
+
)
|
66
|
+
end
|
67
|
+
|
56
68
|
ruby2_keywords def create_table(table_name, *args)
|
57
69
|
options = args.last.is_a?(Hash) ? args.last : {}
|
58
70
|
return super if options[:force] || !table_exists?(table_name)
|
@@ -70,23 +82,51 @@ module SafePgMigrations
|
|
70
82
|
end
|
71
83
|
end
|
72
84
|
|
73
|
-
|
85
|
+
def add_check_constraint(table_name, expression, **options)
|
86
|
+
constraint_definition = check_constraint_for table_name,
|
87
|
+
**check_constraint_options(table_name, expression, options)
|
74
88
|
|
75
|
-
|
76
|
-
|
77
|
-
|
89
|
+
return super if constraint_definition.nil?
|
90
|
+
|
91
|
+
SafePgMigrations.say "/!\\ Constraint '#{constraint_definition.name}' already exists. Skipping statement.", true
|
78
92
|
end
|
79
93
|
|
80
|
-
|
94
|
+
def change_column_null(table_name, column_name, null, *)
|
95
|
+
column = column_for(table_name, column_name)
|
96
|
+
|
97
|
+
return super if column.null != null
|
81
98
|
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
99
|
+
SafePgMigrations.say(
|
100
|
+
"/!\\ Column '#{table_name}.#{column.name}' is already set to 'null: #{null}'. Skipping statement.",
|
101
|
+
true
|
102
|
+
)
|
103
|
+
end
|
104
|
+
|
105
|
+
def validate_check_constraint(table_name, **options)
|
106
|
+
constraint_definition = check_constraint_for!(table_name, **options)
|
107
|
+
|
108
|
+
return super unless constraint_definition.validated?
|
109
|
+
|
110
|
+
SafePgMigrations.say "/!\\ Constraint '#{constraint_definition.name}' already validated. Skipping statement.",
|
111
|
+
true
|
112
|
+
end
|
113
|
+
|
114
|
+
def change_column_default(table_name, column_name, default_or_changes)
|
115
|
+
column = column_for(table_name, column_name)
|
116
|
+
|
117
|
+
previous_alter_statement = change_column_default_for_alter(table_name, column_name, column.default)
|
118
|
+
new_alter_statement = change_column_default_for_alter(table_name, column_name, default_or_changes)
|
119
|
+
|
120
|
+
# NOTE: PG change_column_default is already idempotent.
|
121
|
+
# We try to detect it because it still takes an ACCESS EXCLUSIVE lock
|
122
|
+
|
123
|
+
return super if new_alter_statement != previous_alter_statement
|
124
|
+
|
125
|
+
SafePgMigrations.say(
|
126
|
+
"/!\\ Column '#{table_name}.#{column.name}' is already set to 'default: #{column.default}'. " \
|
127
|
+
'Skipping statement.',
|
128
|
+
true
|
129
|
+
)
|
90
130
|
end
|
91
131
|
end
|
92
132
|
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SafePgMigrations
|
4
|
+
module StatementInsurer
|
5
|
+
module AddColumn
|
6
|
+
ruby2_keywords def add_column(table_name, column_name, type, *args)
|
7
|
+
options = args.last.is_a?(Hash) && args.last
|
8
|
+
options ||= {}
|
9
|
+
|
10
|
+
if should_keep_default_implementation?(**options)
|
11
|
+
with_setting(:statement_timeout, SafePgMigrations.config.pg_statement_timeout) { return super }
|
12
|
+
end
|
13
|
+
|
14
|
+
default = options.delete(:default)
|
15
|
+
null = options.delete(:null)
|
16
|
+
|
17
|
+
with_setting(:statement_timeout, SafePgMigrations.config.pg_statement_timeout) do
|
18
|
+
SafePgMigrations.say_method_call(:add_column, table_name, column_name, type, options)
|
19
|
+
super table_name, column_name, type, **options
|
20
|
+
end
|
21
|
+
|
22
|
+
SafePgMigrations.say_method_call(:change_column_default, table_name, column_name, default)
|
23
|
+
change_column_default(table_name, column_name, default)
|
24
|
+
|
25
|
+
SafePgMigrations.say_method_call(:backfill_column_default, table_name, column_name)
|
26
|
+
without_statement_timeout do
|
27
|
+
backfill_column_default(table_name, column_name)
|
28
|
+
end
|
29
|
+
|
30
|
+
change_column_null(table_name, column_name, null) if null == false
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
|
35
|
+
def should_keep_default_implementation?(default: nil, default_value_backfill: :auto, **)
|
36
|
+
default_value_backfill != :update_in_batches || !default ||
|
37
|
+
!SafePgMigrations::Helpers::SatisfiedHelper.satisfies_add_column_update_rows_backfill?
|
38
|
+
end
|
39
|
+
|
40
|
+
def backfill_column_default(table_name, column_name)
|
41
|
+
model = Class.new(ActiveRecord::Base) { self.table_name = table_name }
|
42
|
+
quoted_column_name = quote_column_name(column_name)
|
43
|
+
|
44
|
+
SafePgMigrations::Helpers::BatchOver.new(model).each_batch do |batch|
|
45
|
+
batch
|
46
|
+
.update_all("#{quoted_column_name} = DEFAULT")
|
47
|
+
sleep SafePgMigrations.config.backfill_pause
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -2,47 +2,41 @@
|
|
2
2
|
|
3
3
|
module SafePgMigrations
|
4
4
|
module StatementInsurer
|
5
|
-
|
5
|
+
include AddColumn
|
6
6
|
|
7
|
-
%i[
|
7
|
+
%i[change_column].each do |method|
|
8
8
|
define_method method do |*args, &block|
|
9
|
-
with_setting(:statement_timeout, SafePgMigrations.config.
|
9
|
+
with_setting(:statement_timeout, SafePgMigrations.config.pg_statement_timeout) { super(*args, &block) }
|
10
10
|
end
|
11
11
|
ruby2_keywords method
|
12
12
|
end
|
13
13
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
default = options.delete(:default)
|
19
|
-
null = options.delete(:null)
|
20
|
-
|
21
|
-
if !default.nil? || null == false
|
22
|
-
SafePgMigrations.say_method_call(:add_column, table_name, column_name, type, options)
|
14
|
+
def validate_check_constraint(table_name, **options)
|
15
|
+
without_statement_timeout do
|
16
|
+
super
|
23
17
|
end
|
18
|
+
end
|
24
19
|
|
25
|
-
|
20
|
+
def add_check_constraint(table_name, expression, **options)
|
21
|
+
Helpers::SatisfiedHelper.satisfies_add_check_constraints!
|
22
|
+
return unless supports_check_constraints?
|
26
23
|
|
27
|
-
|
28
|
-
SafePgMigrations.say_method_call(:change_column_default, table_name, column_name, default)
|
29
|
-
change_column_default(table_name, column_name, default)
|
24
|
+
options = check_constraint_options(table_name, expression, options)
|
30
25
|
|
31
|
-
|
32
|
-
|
33
|
-
end
|
26
|
+
SafePgMigrations.say_method_call :add_check_constraint, table_name, expression, **options, validate: false
|
27
|
+
super table_name, expression, **options, validate: false
|
34
28
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
29
|
+
return unless options.fetch(:validate, true)
|
30
|
+
|
31
|
+
SafePgMigrations.say_method_call :validate_check_constraint, table_name, name: options[:name]
|
32
|
+
validate_check_constraint table_name, name: options[:name]
|
39
33
|
end
|
40
34
|
|
41
35
|
ruby2_keywords def add_foreign_key(from_table, to_table, *args)
|
42
36
|
options = args.last.is_a?(Hash) ? args.last : {}
|
43
|
-
validate_present = options.key?
|
37
|
+
validate_present = options.key?(:validate)
|
44
38
|
options[:validate] = false unless validate_present
|
45
|
-
with_setting(:statement_timeout, SafePgMigrations.config.
|
39
|
+
with_setting(:statement_timeout, SafePgMigrations.config.pg_statement_timeout) do
|
46
40
|
super(from_table, to_table, **options)
|
47
41
|
end
|
48
42
|
|
@@ -53,7 +47,7 @@ module SafePgMigrations
|
|
53
47
|
end
|
54
48
|
|
55
49
|
ruby2_keywords def create_table(*)
|
56
|
-
with_setting(:statement_timeout, SafePgMigrations.config.
|
50
|
+
with_setting(:statement_timeout, SafePgMigrations.config.pg_statement_timeout) do
|
57
51
|
super do |td|
|
58
52
|
yield td if block_given?
|
59
53
|
td.indexes.map! do |key, index_options|
|
@@ -64,7 +58,9 @@ module SafePgMigrations
|
|
64
58
|
end
|
65
59
|
end
|
66
60
|
|
67
|
-
def add_index(table_name, column_name,
|
61
|
+
ruby2_keywords def add_index(table_name, column_name, *args_options)
|
62
|
+
options = args_options.last.is_a?(Hash) ? args_options.last : {}
|
63
|
+
|
68
64
|
if options[:algorithm] == :default
|
69
65
|
options.delete :algorithm
|
70
66
|
else
|
@@ -84,13 +80,20 @@ module SafePgMigrations
|
|
84
80
|
without_timeout { super(table_name, **options) }
|
85
81
|
end
|
86
82
|
|
87
|
-
def
|
88
|
-
|
89
|
-
|
83
|
+
def change_column_null(table_name, column_name, null, default = nil)
|
84
|
+
if default || null || !Helpers::SatisfiedHelper.satisfies_change_column_null_requirements?
|
85
|
+
with_setting(:statement_timeout, SafePgMigrations.config.pg_statement_timeout) { return super }
|
86
|
+
end
|
90
87
|
|
91
|
-
|
92
|
-
|
88
|
+
add_check_constraint table_name, "#{column_name} IS NOT NULL"
|
89
|
+
|
90
|
+
SafePgMigrations.say_method_call :change_column_null, table_name, column_name, false
|
91
|
+
with_setting(:statement_timeout, SafePgMigrations.config.pg_statement_timeout) do
|
92
|
+
super table_name, column_name, false
|
93
93
|
end
|
94
|
+
|
95
|
+
SafePgMigrations.say_method_call :remove_check_constraint, table_name, "#{column_name} IS NOT NULL"
|
96
|
+
remove_check_constraint table_name, "#{column_name} IS NOT NULL"
|
94
97
|
end
|
95
98
|
|
96
99
|
def with_setting(key, value)
|
@@ -109,16 +112,16 @@ module SafePgMigrations
|
|
109
112
|
end
|
110
113
|
end
|
111
114
|
|
112
|
-
def without_statement_timeout
|
113
|
-
with_setting(:statement_timeout, 0)
|
115
|
+
def without_statement_timeout(&block)
|
116
|
+
with_setting(:statement_timeout, 0, &block)
|
114
117
|
end
|
115
118
|
|
116
|
-
def without_lock_timeout
|
117
|
-
with_setting(:lock_timeout, 0)
|
119
|
+
def without_lock_timeout(&block)
|
120
|
+
with_setting(:lock_timeout, 0, &block)
|
118
121
|
end
|
119
122
|
|
120
|
-
def without_timeout
|
121
|
-
without_statement_timeout { without_lock_timeout
|
123
|
+
def without_timeout(&block)
|
124
|
+
without_statement_timeout { without_lock_timeout(&block) }
|
122
125
|
end
|
123
126
|
end
|
124
127
|
end
|
@@ -3,7 +3,7 @@
|
|
3
3
|
module SafePgMigrations
|
4
4
|
module StatementRetrier
|
5
5
|
RETRIABLE_SCHEMA_STATEMENTS = %i[
|
6
|
-
add_column add_foreign_key remove_foreign_key change_column_default change_column_null remove_column
|
6
|
+
add_column add_foreign_key remove_foreign_key change_column_default change_column_null remove_column drop_table
|
7
7
|
].freeze
|
8
8
|
|
9
9
|
RETRIABLE_SCHEMA_STATEMENTS.each do |method|
|
@@ -29,6 +29,14 @@ module SafePgMigrations
|
|
29
29
|
super
|
30
30
|
end
|
31
31
|
|
32
|
+
ruby2_keywords def add_check_constraint(*args)
|
33
|
+
options = args.last.is_a?(Hash) ? args.last : {}
|
34
|
+
if options[:validate] == false
|
35
|
+
UselessStatementsLogger.warn_useless '`validate: :false`', 'https://github.com/doctolib/safe-pg-migrations#safe_add_check_constraint'
|
36
|
+
end
|
37
|
+
super
|
38
|
+
end
|
39
|
+
|
32
40
|
def warn_for_index(**options)
|
33
41
|
return unless options[:algorithm] == :concurrently
|
34
42
|
|
@@ -4,7 +4,7 @@ module SafePgMigrations
|
|
4
4
|
class VerboseSqlLogger
|
5
5
|
def setup
|
6
6
|
@activerecord_logger_was = ActiveRecord::Base.logger
|
7
|
-
@verbose_query_logs_was =
|
7
|
+
@verbose_query_logs_was = Polyfills::VerboseQueryLogsPolyfill.verbose_query_logs
|
8
8
|
@colorize_logging_was = ActiveRecord::LogSubscriber.colorize_logging
|
9
9
|
|
10
10
|
disable_marginalia if defined?(Marginalia)
|
@@ -13,12 +13,12 @@ module SafePgMigrations
|
|
13
13
|
ActiveRecord::Base.logger = stdout_logger
|
14
14
|
ActiveRecord::LogSubscriber.colorize_logging = colorize_logging?
|
15
15
|
# Do not output caller method, we know it is coming from the migration
|
16
|
-
|
16
|
+
Polyfills::VerboseQueryLogsPolyfill.verbose_query_logs = false
|
17
17
|
self
|
18
18
|
end
|
19
19
|
|
20
20
|
def teardown
|
21
|
-
|
21
|
+
Polyfills::VerboseQueryLogsPolyfill.verbose_query_logs = @verbose_query_logs_was
|
22
22
|
ActiveRecord::LogSubscriber.colorize_logging = @colorize_logging_was
|
23
23
|
ActiveRecord::Base.logger = @activerecord_logger_was
|
24
24
|
enable_marginalia if defined?(Marginalia)
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SafePgMigrations
|
4
|
+
module Polyfills
|
5
|
+
module IndexDefinitionPolyfill
|
6
|
+
include Helpers::SatisfiedHelper
|
7
|
+
|
8
|
+
protected
|
9
|
+
|
10
|
+
IndexDefinition = Struct.new(:table, :name)
|
11
|
+
|
12
|
+
def index_definition(table_name, column_name, **options)
|
13
|
+
return super(table_name, column_name, **options) if Helpers::SatisfiedHelper.satisfied? '>=6.1.0'
|
14
|
+
|
15
|
+
index_name = options.key?(:name) ? options[:name].to_s : index_name(table_name, index_column_names(column_name))
|
16
|
+
validate_index_length!(table_name, index_name, options.fetch(:internal, false))
|
17
|
+
|
18
|
+
IndexDefinition.new(table_name, index_name)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SafePgMigrations
|
4
|
+
module Polyfills
|
5
|
+
module VerboseQueryLogsPolyfill
|
6
|
+
class << self
|
7
|
+
def verbose_query_logs
|
8
|
+
return ActiveRecord.verbose_query_logs if Helpers::SatisfiedHelper.satisfied? '>=7.0.0'
|
9
|
+
|
10
|
+
ActiveRecord::Base.verbose_query_logs
|
11
|
+
end
|
12
|
+
|
13
|
+
def verbose_query_logs=(value)
|
14
|
+
if Helpers::SatisfiedHelper.satisfied? '>=7.0.0'
|
15
|
+
ActiveRecord.verbose_query_logs = value
|
16
|
+
return
|
17
|
+
end
|
18
|
+
|
19
|
+
ActiveRecord::Base.verbose_query_logs = value
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -7,6 +7,7 @@ module SafePgMigrations
|
|
7
7
|
initializer 'safe_pg_migrations.insert_into_active_record' do
|
8
8
|
ActiveSupport.on_load :active_record do
|
9
9
|
ActiveRecord::Migration.prepend(SafePgMigrations::Migration)
|
10
|
+
ActiveRecord::Migration.singleton_class.prepend(SafePgMigrations::Migration::ClassMethods)
|
10
11
|
end
|
11
12
|
end
|
12
13
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: safe-pg-migrations
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 2.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Matthieu Prat
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date:
|
13
|
+
date: 2023-06-29 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: activerecord
|
@@ -18,42 +18,28 @@ dependencies:
|
|
18
18
|
requirements:
|
19
19
|
- - ">="
|
20
20
|
- !ruby/object:Gem::Version
|
21
|
-
version: '
|
21
|
+
version: '6.0'
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
24
|
version_requirements: !ruby/object:Gem::Requirement
|
25
25
|
requirements:
|
26
26
|
- - ">="
|
27
27
|
- !ruby/object:Gem::Version
|
28
|
-
version: '
|
28
|
+
version: '6.0'
|
29
29
|
- !ruby/object:Gem::Dependency
|
30
30
|
name: activesupport
|
31
31
|
requirement: !ruby/object:Gem::Requirement
|
32
32
|
requirements:
|
33
33
|
- - ">="
|
34
34
|
- !ruby/object:Gem::Version
|
35
|
-
version: '
|
35
|
+
version: '6.0'
|
36
36
|
type: :runtime
|
37
37
|
prerelease: false
|
38
38
|
version_requirements: !ruby/object:Gem::Requirement
|
39
39
|
requirements:
|
40
40
|
- - ">="
|
41
41
|
- !ruby/object:Gem::Version
|
42
|
-
version: '
|
43
|
-
- !ruby/object:Gem::Dependency
|
44
|
-
name: ruby2_keywords
|
45
|
-
requirement: !ruby/object:Gem::Requirement
|
46
|
-
requirements:
|
47
|
-
- - ">="
|
48
|
-
- !ruby/object:Gem::Version
|
49
|
-
version: 0.0.4
|
50
|
-
type: :runtime
|
51
|
-
prerelease: false
|
52
|
-
version_requirements: !ruby/object:Gem::Requirement
|
53
|
-
requirements:
|
54
|
-
- - ">="
|
55
|
-
- !ruby/object:Gem::Version
|
56
|
-
version: 0.0.4
|
42
|
+
version: '6.0'
|
57
43
|
description: Make your PG migrations safe.
|
58
44
|
email:
|
59
45
|
executables: []
|
@@ -65,13 +51,20 @@ files:
|
|
65
51
|
- lib/safe-pg-migrations.rb
|
66
52
|
- lib/safe-pg-migrations/base.rb
|
67
53
|
- lib/safe-pg-migrations/configuration.rb
|
54
|
+
- lib/safe-pg-migrations/helpers/batch_over.rb
|
55
|
+
- lib/safe-pg-migrations/helpers/blocking_activity_formatter.rb
|
56
|
+
- lib/safe-pg-migrations/helpers/blocking_activity_selector.rb
|
57
|
+
- lib/safe-pg-migrations/helpers/index_helper.rb
|
58
|
+
- lib/safe-pg-migrations/helpers/satisfied_helper.rb
|
68
59
|
- lib/safe-pg-migrations/plugins/blocking_activity_logger.rb
|
69
60
|
- lib/safe-pg-migrations/plugins/idempotent_statements.rb
|
70
|
-
- lib/safe-pg-migrations/plugins/legacy_active_record_support.rb
|
71
61
|
- lib/safe-pg-migrations/plugins/statement_insurer.rb
|
62
|
+
- lib/safe-pg-migrations/plugins/statement_insurer/add_column.rb
|
72
63
|
- lib/safe-pg-migrations/plugins/statement_retrier.rb
|
73
64
|
- lib/safe-pg-migrations/plugins/useless_statements_logger.rb
|
74
65
|
- lib/safe-pg-migrations/plugins/verbose_sql_logger.rb
|
66
|
+
- lib/safe-pg-migrations/polyfills/index_definition_polyfill.rb
|
67
|
+
- lib/safe-pg-migrations/polyfills/verbose_query_logs_polyfill.rb
|
75
68
|
- lib/safe-pg-migrations/railtie.rb
|
76
69
|
- lib/safe-pg-migrations/version.rb
|
77
70
|
homepage: https://github.com/doctolib/safe-pg-migrations
|
@@ -83,6 +76,7 @@ metadata:
|
|
83
76
|
mailing_list_uri: https://doctolib.engineering/engineering-news-ruby-rails-react
|
84
77
|
source_code_uri: https://github.com/doctolib/safe-pg-migrations
|
85
78
|
contributors_uri: https://github.com/doctolib/safe-pg-migrations/graphs/contributors
|
79
|
+
rubygems_mfa_required: 'true'
|
86
80
|
post_install_message:
|
87
81
|
rdoc_options: []
|
88
82
|
require_paths:
|
@@ -91,18 +85,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
91
85
|
requirements:
|
92
86
|
- - ">="
|
93
87
|
- !ruby/object:Gem::Version
|
94
|
-
version: '2.
|
95
|
-
- - "<"
|
96
|
-
- !ruby/object:Gem::Version
|
97
|
-
version: '4'
|
88
|
+
version: '2.7'
|
98
89
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
99
90
|
requirements:
|
100
91
|
- - ">="
|
101
92
|
- !ruby/object:Gem::Version
|
102
93
|
version: '0'
|
103
94
|
requirements: []
|
104
|
-
|
105
|
-
rubygems_version: 2.7.3
|
95
|
+
rubygems_version: 3.3.7
|
106
96
|
signing_key:
|
107
97
|
specification_version: 4
|
108
98
|
summary: Make your PG migrations safe.
|
@@ -1,38 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module SafePgMigrations
|
4
|
-
module LegacyActiveRecordSupport
|
5
|
-
ruby2_keywords def validate_foreign_key(*args)
|
6
|
-
return super(*args) if satisfied? '>=6.0.0'
|
7
|
-
|
8
|
-
from_table, to_table, options = args
|
9
|
-
super(from_table, to_table || options)
|
10
|
-
end
|
11
|
-
|
12
|
-
ruby2_keywords def foreign_key_exists?(*args)
|
13
|
-
return super(*args) if satisfied? '>=6.0.0'
|
14
|
-
|
15
|
-
from_table, to_table, options = args
|
16
|
-
super(from_table, to_table || options)
|
17
|
-
end
|
18
|
-
|
19
|
-
protected
|
20
|
-
|
21
|
-
IndexDefinition = Struct.new(:table, :name)
|
22
|
-
|
23
|
-
def index_definition(table_name, column_name, **options)
|
24
|
-
return super(table_name, column_name, **options) if satisfied? '>=6.1.0'
|
25
|
-
|
26
|
-
index_name = options.key?(:name) ? options[:name].to_s : index_name(table_name, index_column_names(column_name))
|
27
|
-
validate_index_length!(table_name, index_name, options.fetch(:internal, false))
|
28
|
-
|
29
|
-
IndexDefinition.new(table_name, index_name)
|
30
|
-
end
|
31
|
-
|
32
|
-
private
|
33
|
-
|
34
|
-
def satisfied?(version)
|
35
|
-
Gem::Requirement.new(version).satisfied_by? Gem::Version.new(::ActiveRecord::VERSION::STRING)
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|