safe-pg-migrations 1.4.2 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +115 -33
- data/lib/safe-pg-migrations/base.rb +25 -9
- data/lib/safe-pg-migrations/configuration.rb +15 -9
- data/lib/safe-pg-migrations/helpers/batch_over.rb +56 -0
- data/lib/safe-pg-migrations/helpers/blocking_activity_formatter.rb +59 -0
- data/lib/safe-pg-migrations/helpers/blocking_activity_selector.rb +54 -0
- data/lib/safe-pg-migrations/helpers/index_helper.rb +24 -0
- data/lib/safe-pg-migrations/helpers/satisfied_helper.rb +31 -0
- data/lib/safe-pg-migrations/plugins/blocking_activity_logger.rb +51 -80
- data/lib/safe-pg-migrations/plugins/idempotent_statements.rb +53 -13
- data/lib/safe-pg-migrations/plugins/statement_insurer/add_column.rb +52 -0
- data/lib/safe-pg-migrations/plugins/statement_insurer.rb +41 -38
- data/lib/safe-pg-migrations/plugins/statement_retrier.rb +1 -1
- data/lib/safe-pg-migrations/plugins/useless_statements_logger.rb +8 -0
- data/lib/safe-pg-migrations/plugins/verbose_sql_logger.rb +3 -3
- data/lib/safe-pg-migrations/polyfills/index_definition_polyfill.rb +22 -0
- data/lib/safe-pg-migrations/polyfills/verbose_query_logs_polyfill.rb +24 -0
- data/lib/safe-pg-migrations/railtie.rb +1 -0
- data/lib/safe-pg-migrations/version.rb +1 -1
- metadata +17 -27
- data/lib/safe-pg-migrations/plugins/legacy_active_record_support.rb +0 -38
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2fafd98448f0ff697da386a337dc22753be2144feedba4793f87a351581a5607
|
4
|
+
data.tar.gz: 9653ae6395d4c9f27b69dba9dee9142a4535384c42e8d05fac9ca2e2ef1bdce6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 64297c9ba4f040e7afa21fefa2acb2a26e27fb0bfee3d9f9cbf902ee4bb0ac087f4e22591a7b5470cf73dc7bc73345e59904649c02d7540bd4693372e7069549
|
7
|
+
data.tar.gz: 7a2640fd2494480d560cfa6f19059550ecc316eaf729df6d8f97a434c189f4565c49d47acbb02f437d9b1285a466a5615a79a84d6ba1dfb3d94bb54f87b9e788
|
data/README.md
CHANGED
@@ -6,9 +6,9 @@ ActiveRecord migrations for Postgres made safe.
|
|
6
6
|
|
7
7
|
## Requirements
|
8
8
|
|
9
|
-
- Ruby 2.
|
10
|
-
- Rails
|
11
|
-
- PostgreSQL
|
9
|
+
- Ruby 2.7+
|
10
|
+
- Rails 6.0+
|
11
|
+
- PostgreSQL 11.7+
|
12
12
|
|
13
13
|
## Usage
|
14
14
|
|
@@ -25,43 +25,46 @@ gem 'safe-pg-migrations'
|
|
25
25
|
Consider the following migration:
|
26
26
|
|
27
27
|
```rb
|
28
|
-
class
|
28
|
+
class AddPatientRefToAppointments < ActiveRecord::Migration[6.0]
|
29
29
|
def change
|
30
|
-
|
30
|
+
add_reference :appointments, :patient
|
31
31
|
end
|
32
32
|
end
|
33
33
|
```
|
34
34
|
|
35
|
-
If the `users` table is large, running this migration
|
35
|
+
If the `users` table is large, running this migration will likely cause downtime. **Safe PG Migrations** hooks into Active Record so that the following gets executed instead:
|
36
36
|
|
37
37
|
```rb
|
38
|
-
class
|
38
|
+
class AddPatientRefToAppointments < ActiveRecord::Migration[6.0]
|
39
39
|
# Do not wrap the migration in a transaction so that locks are held for a shorter time.
|
40
40
|
disable_ddl_transaction!
|
41
41
|
|
42
42
|
def change
|
43
43
|
# Lower Postgres' lock timeout to avoid statement queueing. Acts like a seatbelt.
|
44
|
-
execute
|
44
|
+
execute("SET lock_timeout TO '5s'")
|
45
45
|
|
46
|
-
#
|
47
|
-
|
46
|
+
# Lower Postgres' statement timeout to avoid too long transactions. Acts like a seatbelt.
|
47
|
+
execute("SET statement_timeout TO '5s'")
|
48
|
+
add_column :appointments, :patient_id, :bigint
|
48
49
|
|
49
|
-
#
|
50
|
-
|
50
|
+
# add_index using the concurrent algorithm, to avoid locking the tables
|
51
|
+
add_index :appointments, :patient_id, algorithm: :concurrently
|
51
52
|
|
52
|
-
#
|
53
|
-
|
53
|
+
# add_foreign_key without validation, to avoid locking the table for too long
|
54
|
+
execute("SET statement_timeout TO '5s'")
|
55
|
+
add_foreign_key :appointments, :patients, validate: false
|
54
56
|
|
55
|
-
|
56
|
-
|
57
|
-
# the
|
58
|
-
|
59
|
-
|
57
|
+
execute("SET statement_timeout TO '0'")
|
58
|
+
|
59
|
+
# validate the foreign key separately, it avoids taking a lock on the entire tables
|
60
|
+
validate_foreign_key :appointments, :patients
|
61
|
+
|
62
|
+
# we also need to set timeouts to their initial values if needed
|
60
63
|
end
|
61
64
|
end
|
62
65
|
```
|
63
66
|
|
64
|
-
Under the hood, **Safe PG Migrations** patches `ActiveRecord::Migration` and extends `ActiveRecord::Base.connection` to make potentially dangerous methods—like `
|
67
|
+
Under the hood, **Safe PG Migrations** patches `ActiveRecord::Migration` and extends `ActiveRecord::Base.connection` to make potentially dangerous methods—like `add_reference`—safe.
|
65
68
|
|
66
69
|
## Motivation
|
67
70
|
|
@@ -106,23 +109,43 @@ Beware though, when adding a volatile default value:
|
|
106
109
|
```ruby
|
107
110
|
add_column :users, :created_at, default: 'clock_timestamp()'
|
108
111
|
```
|
109
|
-
PG will still needs to update every row of the table, and will most likely statement timeout for big table. In this case,
|
112
|
+
PG will still needs to update every row of the table, and will most likely statement timeout for big table. In this case, **Safe PG Migrations** can automatically backfill data when the option `default_value_backfill:` is set to `:update_in_batches`.
|
113
|
+
|
114
|
+
</details>
|
115
|
+
|
116
|
+
<details>
|
117
|
+
<summary>Safe add_column - adding a volatile default value</summary>
|
118
|
+
|
119
|
+
**Safe PG Migrations** provides the extra option parameter `default_value_backfill:`. When your migration is adding a volatile default value, the option `:update_in_batches` can be set. It will automatically backfill the value in a safe manner.
|
120
|
+
|
121
|
+
```ruby
|
122
|
+
add_column :users, :created_at, default: 'clock_timestamp()', default_value_backfill: :update_in_batches
|
123
|
+
```
|
124
|
+
|
125
|
+
More specifically, it will:
|
126
|
+
|
127
|
+
1. create the column without default value and without null constraint. This ensure the `ACCESS EXCLUSIVE` lock is acquired for the least amount of time;
|
128
|
+
2. add the default value, without data backfill. An `ACCESS EXCLUSIVE` lock is acquired and released immediately;
|
129
|
+
3. backfill data, in batch of `SafePgMigrations.config.backfill_batch_size` and with a pause of `SafePgMigrations.config.backfill_pause` between each batch;
|
130
|
+
4. change the column to `null: false`, if defined in the parameters, following the algorithm we have defined below.
|
110
131
|
|
111
|
-
|
132
|
+
---
|
133
|
+
**NOTE**
|
112
134
|
|
113
|
-
|
114
|
-
Adding a column with a default value and a not-null constraint is [dangerous](https://wework.github.io/data/2015/11/05/add-columns-with-default-values-to-large-tables-in-rails-postgres/).
|
135
|
+
Data backfill take time. If your table is big, your migrations will (safely) hangs for a while. You might want to backfill data manually instead, to do so you will need two migrations
|
115
136
|
|
116
|
-
|
137
|
+
1. First migration :
|
117
138
|
|
118
|
-
|
119
|
-
2. Then set the default value on the column,
|
120
|
-
3. Then backfilling the column,
|
121
|
-
4. And then adding the not null constraint with a short statement timeout.
|
139
|
+
a. adds the column without default and without null constraint;
|
122
140
|
|
123
|
-
|
141
|
+
b. add the default value.
|
124
142
|
|
125
|
-
|
143
|
+
2. manual data backfill (rake task, manual operation, ...)
|
144
|
+
3. Second migration which change the column to null false (with **Safe PG Migrations**, `change_column_null` is safe and can be used; see section below)
|
145
|
+
|
146
|
+
---
|
147
|
+
|
148
|
+
`default_value_backfill:` also accept the value `:auto` which is set by default. In this case, **Safe PG Migrations** will not backfill data and will let PostgreSQL handle it itself.
|
126
149
|
|
127
150
|
</details>
|
128
151
|
|
@@ -143,13 +166,70 @@ If you still get lock timeout while adding / removing indexes, it might be for o
|
|
143
166
|
|
144
167
|
Adding a foreign key requires a `SHARE ROW EXCLUSIVE` lock, which **prevent writing in the tables** while the migration is running.
|
145
168
|
|
146
|
-
Adding the constraint itself is rather fast, the major part of the time is spent on validating this constraint. Thus
|
169
|
+
Adding the constraint itself is rather fast, the major part of the time is spent on validating this constraint. Thus **Safe PG Migrations** ensures that adding a foreign key holds blocking locks for the least amount of time by splitting the foreign key creation in two steps:
|
147
170
|
|
148
171
|
1. adding the constraint *without validation*, will not validate existing rows;
|
149
172
|
2. validating the constraint, will validate existing rows in the table, without blocking read or write on the table
|
150
173
|
|
151
174
|
</details>
|
152
175
|
|
176
|
+
|
177
|
+
<details><summary id="safe_add_check_constraint">Safe <code>add_check_constraint</code> (ActiveRecord > 6.1)</summary>
|
178
|
+
|
179
|
+
Adding a check constraint requires an `ACCESS EXCLUSIVE` lock, which **prevent writing and reading in the tables** [as soon as the lock is requested](https://medium.com/doctolib/stop-worrying-about-postgresql-locks-in-your-rails-migrations-3426027e9cc9).
|
180
|
+
|
181
|
+
Adding the constraint itself is rather fast, the major part of the time is spent on validating this constraint.
|
182
|
+
Thus **Safe PG Migrations** ensures that adding a constraints holds blocking locks for the least amount of time by
|
183
|
+
splitting the constraint addition in two steps:
|
184
|
+
|
185
|
+
1. adding the constraint *without validation*, will not validate existing rows;
|
186
|
+
2. validating the constraint, will validate existing rows in the table, without blocking read or write on the table
|
187
|
+
|
188
|
+
</details>
|
189
|
+
|
190
|
+
<details><summary id="safe_change_column_null">Safe <code>change_column_null</code> (ActiveRecord and PG version dependant)</summary>
|
191
|
+
|
192
|
+
Changing the nullability of a column requires an `ACCESS EXCLUSIVE` lock, which **prevent writing and reading in the tables** [as soon as the lock is requested](https://medium.com/doctolib/stop-worrying-about-postgresql-locks-in-your-rails-migrations-3426027e9cc9).
|
193
|
+
|
194
|
+
Adding the constraint itself is rather fast, the major part of the time is spent on validating this constraint.
|
195
|
+
|
196
|
+
**Safe PG Migrations** acts differently depending on the version you are on.
|
197
|
+
|
198
|
+
### Recent versions of PG and Active Record (> 12 and > 6.1)
|
199
|
+
|
200
|
+
Starting on PostgreSQL versions 12, adding the column NOT NULL constraint is safe if a check constraint validates the
|
201
|
+
nullability of the same column. **Safe PG Migrations** also relies on add_check_constraint, which was introduced in
|
202
|
+
ActiveRecord 6.1.
|
203
|
+
|
204
|
+
If these requirements are met, **Safe PG Migrations** ensures that adding a constraints holds blocking locks for the least
|
205
|
+
amount of time by splitting the constraint addition in several steps:
|
206
|
+
|
207
|
+
1. adding a `IS NOT NULL` constraint *without validation*, will not validate existing rows but block read or write;
|
208
|
+
2. validating the constraint, will validate existing rows in the table, without blocking read or write on the table;
|
209
|
+
3. changing the not null status of the column, thanks to the NOT NULL constraint without having to scan the table sequentially;
|
210
|
+
4. dropping the `IS NOT NULL` constraint.
|
211
|
+
|
212
|
+
### Older versions of PG or ActiveRecord
|
213
|
+
|
214
|
+
If the version of PostgreSQL is below 12, or if the version of ActiveRecord is below 6.1, **Safe PG Migrations** will only
|
215
|
+
wrap ActiveRecord method into a statement timeout and lock timeout.
|
216
|
+
|
217
|
+
### Call with a default parameter
|
218
|
+
|
219
|
+
Calling change_column_null with a default parameter [is dangerous](https://github.com/rails/rails/blob/716baea69f989b64f5bfeaff880c2512377bebab/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb#L446)
|
220
|
+
and is likely not to finish in the statement timeout defined by **Safe PG Migrations**. For this reason, when the default
|
221
|
+
parameter is given, **Safe PG Migrations** will simply forward it to activerecord methods without trying to improve it
|
222
|
+
|
223
|
+
### Dropping a NULL constraint
|
224
|
+
|
225
|
+
Dropping a null constraint still requires an `ACCESS EXCLUSIVE` lock, but does not require extra operation to reduce the
|
226
|
+
amount of time during which the lock is held. **Safe PG Migrations** only wrap methods of activerecord in lock and statement
|
227
|
+
timeouts
|
228
|
+
|
229
|
+
</details>
|
230
|
+
|
231
|
+
|
232
|
+
|
153
233
|
<details><summary>Retry after lock timeout</summary>
|
154
234
|
|
155
235
|
When a statement fails with a lock timeout, **Safe PG Migrations** retries it (5 times max) [list of retriable statements](https://github.com/doctolib/safe-pg-migrations/blob/66933256252b6bbf12e404b829a361dbba30e684/lib/safe-pg-migrations/plugins/statement_retrier.rb#L5)
|
@@ -214,7 +294,9 @@ SafePgMigrations.config.blocking_activity_logger_verbose = true # Outputs the ra
|
|
214
294
|
|
215
295
|
SafePgMigrations.config.blocking_activity_logger_margin = 1.second # Delay to output blocking queries before timeout. Must be shorter than safe_timeout
|
216
296
|
|
217
|
-
SafePgMigrations.config.
|
297
|
+
SafePgMigrations.config.backfill_batch_size = 100_000 # Size of the batches used for backfilling when adding a column with a default value
|
298
|
+
|
299
|
+
SafePgMigrations.config.backfill_pause = 0.5.second # Delay between each batch during a backfill. This ensure replication can happen safely.
|
218
300
|
|
219
301
|
SafePgMigrations.config.retry_delay = 1.minute # Delay between retries for retryable statements
|
220
302
|
|
@@ -1,14 +1,18 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require 'ruby2_keywords'
|
4
3
|
require 'safe-pg-migrations/configuration'
|
4
|
+
require 'safe-pg-migrations/helpers/satisfied_helper'
|
5
|
+
require 'safe-pg-migrations/helpers/index_helper'
|
6
|
+
require 'safe-pg-migrations/helpers/batch_over'
|
5
7
|
require 'safe-pg-migrations/plugins/verbose_sql_logger'
|
6
8
|
require 'safe-pg-migrations/plugins/blocking_activity_logger'
|
9
|
+
require 'safe-pg-migrations/plugins/statement_insurer/add_column'
|
7
10
|
require 'safe-pg-migrations/plugins/statement_insurer'
|
8
11
|
require 'safe-pg-migrations/plugins/statement_retrier'
|
9
12
|
require 'safe-pg-migrations/plugins/idempotent_statements'
|
10
13
|
require 'safe-pg-migrations/plugins/useless_statements_logger'
|
11
|
-
require 'safe-pg-migrations/
|
14
|
+
require 'safe-pg-migrations/polyfills/index_definition_polyfill'
|
15
|
+
require 'safe-pg-migrations/polyfills/verbose_query_logs_polyfill'
|
12
16
|
|
13
17
|
module SafePgMigrations
|
14
18
|
# Order matters: the bottom-most plugin will have precedence
|
@@ -18,30 +22,26 @@ module SafePgMigrations
|
|
18
22
|
StatementRetrier,
|
19
23
|
StatementInsurer,
|
20
24
|
UselessStatementsLogger,
|
21
|
-
|
25
|
+
Polyfills::IndexDefinitionPolyfill,
|
22
26
|
].freeze
|
23
27
|
|
24
28
|
class << self
|
25
29
|
attr_reader :current_migration, :pg_version_num
|
26
30
|
|
27
|
-
def setup_and_teardown(migration, connection)
|
31
|
+
def setup_and_teardown(migration, connection, &block)
|
28
32
|
@pg_version_num = get_pg_version_num(connection)
|
29
33
|
@alternate_connection = nil
|
30
34
|
@current_migration = migration
|
31
35
|
stdout_sql_logger = VerboseSqlLogger.new.setup if verbose?
|
32
36
|
PLUGINS.each { |plugin| connection.extend(plugin) }
|
33
37
|
|
34
|
-
connection.with_setting
|
38
|
+
connection.with_setting :lock_timeout, SafePgMigrations.config.pg_lock_timeout, &block
|
35
39
|
ensure
|
36
40
|
close_alternate_connection
|
37
41
|
@current_migration = nil
|
38
42
|
stdout_sql_logger&.teardown
|
39
43
|
end
|
40
44
|
|
41
|
-
def get_pg_version_num(connection)
|
42
|
-
connection.query_value('SHOW server_version_num').to_i
|
43
|
-
end
|
44
|
-
|
45
45
|
def alternate_connection
|
46
46
|
@alternate_connection ||= ActiveRecord::Base.connection_pool.send(:new_connection)
|
47
47
|
end
|
@@ -64,6 +64,9 @@ module SafePgMigrations
|
|
64
64
|
end
|
65
65
|
|
66
66
|
def verbose?
|
67
|
+
unless current_migration.class._safe_pg_migrations_verbose.nil?
|
68
|
+
return current_migration.class._safe_pg_migrations_verbose
|
69
|
+
end
|
67
70
|
return ENV['SAFE_PG_MIGRATIONS_VERBOSE'] == '1' if ENV['SAFE_PG_MIGRATIONS_VERBOSE']
|
68
71
|
return Rails.env.production? if defined?(Rails)
|
69
72
|
|
@@ -73,9 +76,21 @@ module SafePgMigrations
|
|
73
76
|
def config
|
74
77
|
@config ||= Configuration.new
|
75
78
|
end
|
79
|
+
|
80
|
+
def get_pg_version_num(connection)
|
81
|
+
connection.query_value('SHOW server_version_num').to_i
|
82
|
+
end
|
76
83
|
end
|
77
84
|
|
78
85
|
module Migration
|
86
|
+
module ClassMethods
|
87
|
+
attr_accessor :_safe_pg_migrations_verbose
|
88
|
+
|
89
|
+
def safe_pg_migrations_verbose(verbose)
|
90
|
+
@_safe_pg_migrations_verbose = verbose
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
79
94
|
def exec_migration(connection, direction)
|
80
95
|
SafePgMigrations.setup_and_teardown(self, connection) do
|
81
96
|
super(connection, direction)
|
@@ -95,6 +110,7 @@ module SafePgMigrations
|
|
95
110
|
add_belongs_to
|
96
111
|
change_column_null
|
97
112
|
add_foreign_key
|
113
|
+
add_check_constraint
|
98
114
|
].freeze
|
99
115
|
|
100
116
|
SAFE_METHODS.each do |method|
|
@@ -4,26 +4,32 @@ require 'active_support/core_ext/numeric/time'
|
|
4
4
|
|
5
5
|
module SafePgMigrations
|
6
6
|
class Configuration
|
7
|
-
attr_accessor :safe_timeout
|
8
|
-
|
9
|
-
attr_accessor :blocking_activity_logger_verbose
|
10
|
-
attr_accessor :batch_size
|
11
|
-
attr_accessor :retry_delay
|
12
|
-
attr_accessor :max_tries
|
7
|
+
attr_accessor :safe_timeout, :blocking_activity_logger_margin, :blocking_activity_logger_verbose,
|
8
|
+
:backfill_batch_size, :backfill_pause, :retry_delay, :max_tries
|
13
9
|
|
14
10
|
def initialize
|
15
11
|
self.safe_timeout = 5.seconds
|
16
12
|
self.blocking_activity_logger_margin = 1.second
|
17
13
|
self.blocking_activity_logger_verbose = true
|
18
|
-
self.
|
14
|
+
self.backfill_batch_size = 100_000
|
15
|
+
self.backfill_pause = 0.5.second
|
19
16
|
self.retry_delay = 1.minute
|
20
17
|
self.max_tries = 5
|
21
18
|
end
|
22
19
|
|
23
|
-
def
|
24
|
-
pg_duration
|
20
|
+
def pg_statement_timeout
|
21
|
+
pg_duration safe_timeout
|
25
22
|
end
|
26
23
|
|
24
|
+
def pg_lock_timeout
|
25
|
+
# if statement timeout and lock timeout have the same value, statement timeout will raise in priority. We actually
|
26
|
+
# need the opposite for BlockingActivityLogger to detect lock timeouts correctly.
|
27
|
+
# By reducing the lock timeout by a very small margin, we ensure that the lock timeout is raised in priority
|
28
|
+
pg_duration safe_timeout * 0.99
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
27
33
|
def pg_duration(duration)
|
28
34
|
value, unit = duration.integer? ? [duration, 's'] : [(duration * 1000).to_i, 'ms']
|
29
35
|
"#{value}#{unit}"
|
@@ -0,0 +1,56 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SafePgMigrations
|
4
|
+
module Helpers
|
5
|
+
class BatchOver
|
6
|
+
def initialize(model, of: SafePgMigrations.config.backfill_batch_size)
|
7
|
+
@model = model
|
8
|
+
@of = of
|
9
|
+
|
10
|
+
@current_range = nil
|
11
|
+
end
|
12
|
+
|
13
|
+
def each_batch
|
14
|
+
yield scope.where(primary_key => @current_range) while next_batch
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
def next_batch
|
20
|
+
return if endless?
|
21
|
+
|
22
|
+
first = next_scope.take
|
23
|
+
|
24
|
+
return unless first
|
25
|
+
|
26
|
+
last = next_scope.offset(@of).take
|
27
|
+
|
28
|
+
first_key = first[primary_key]
|
29
|
+
last_key = last.nil? ? nil : last[primary_key]
|
30
|
+
|
31
|
+
@current_range = first_key...last_key
|
32
|
+
end
|
33
|
+
|
34
|
+
def next_scope
|
35
|
+
return scope if @current_range.nil?
|
36
|
+
return scope.none if endless?
|
37
|
+
|
38
|
+
scope.where(primary_key => @current_range.end..)
|
39
|
+
end
|
40
|
+
|
41
|
+
def scope
|
42
|
+
@model.order(primary_key => :asc)
|
43
|
+
end
|
44
|
+
|
45
|
+
def endless?
|
46
|
+
return false if @current_range.nil?
|
47
|
+
|
48
|
+
@current_range.end.nil?
|
49
|
+
end
|
50
|
+
|
51
|
+
def primary_key
|
52
|
+
@model.primary_key
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SafePgMigrations
|
4
|
+
module Helpers
|
5
|
+
module BlockingActivityFormatter
|
6
|
+
def log_queries(queries)
|
7
|
+
if queries.empty?
|
8
|
+
SafePgMigrations.say 'Could not find any blocking query.', true
|
9
|
+
else
|
10
|
+
SafePgMigrations.say(
|
11
|
+
"Statement was being blocked by the following #{'query'.pluralize(queries.size)}:",
|
12
|
+
true
|
13
|
+
)
|
14
|
+
SafePgMigrations.say '', true
|
15
|
+
output_blocking_queries(queries)
|
16
|
+
SafePgMigrations.say(
|
17
|
+
'Beware, some of those queries might run in a transaction. In this case the locking query might be ' \
|
18
|
+
'located elsewhere in the transaction',
|
19
|
+
true
|
20
|
+
)
|
21
|
+
SafePgMigrations.say '', true
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
def output_blocking_queries(queries)
|
28
|
+
if SafePgMigrations.config.blocking_activity_logger_verbose
|
29
|
+
queries.each do |pid, query, start_time|
|
30
|
+
SafePgMigrations.say "Query with pid #{pid || 'null'} started #{format_start_time start_time}: #{query}",
|
31
|
+
true
|
32
|
+
end
|
33
|
+
else
|
34
|
+
output_confidentially_blocking_queries(queries)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def output_confidentially_blocking_queries(queries)
|
39
|
+
queries.each do |start_time, locktype, mode, pid, transactionid|
|
40
|
+
SafePgMigrations.say(
|
41
|
+
"Query with pid #{pid || 'null'} " \
|
42
|
+
"started #{format_start_time(start_time)}: " \
|
43
|
+
"lock type: #{locktype || 'null'}, " \
|
44
|
+
"lock mode: #{mode || 'null'}, " \
|
45
|
+
"lock transactionid: #{transactionid || 'null'}",
|
46
|
+
true
|
47
|
+
)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def format_start_time(start_time, reference_time = Time.now)
|
52
|
+
start_time = Time.parse(start_time) unless start_time.is_a? Time
|
53
|
+
|
54
|
+
duration = (reference_time - start_time).round
|
55
|
+
"#{duration} #{'second'.pluralize(duration)} ago"
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SafePgMigrations
|
4
|
+
module Helpers
|
5
|
+
module BlockingActivitySelector
|
6
|
+
FILTERED_COLUMNS = %w[
|
7
|
+
blocked_activity.xact_start
|
8
|
+
blocked_locks.locktype
|
9
|
+
blocked_locks.mode
|
10
|
+
blocking_activity.pid
|
11
|
+
blocked_locks.transactionid
|
12
|
+
].freeze
|
13
|
+
|
14
|
+
VERBOSE_COLUMNS = %w[
|
15
|
+
blocking_activity.pid
|
16
|
+
blocking_activity.query
|
17
|
+
blocked_activity.xact_start
|
18
|
+
].freeze
|
19
|
+
|
20
|
+
def select_blocking_queries_sql
|
21
|
+
columns =
|
22
|
+
(
|
23
|
+
if SafePgMigrations.config.blocking_activity_logger_verbose
|
24
|
+
VERBOSE_COLUMNS
|
25
|
+
else
|
26
|
+
FILTERED_COLUMNS
|
27
|
+
end
|
28
|
+
)
|
29
|
+
|
30
|
+
<<~SQL.squish
|
31
|
+
SELECT #{columns.join(', ')}
|
32
|
+
FROM pg_catalog.pg_locks blocked_locks
|
33
|
+
JOIN pg_catalog.pg_stat_activity blocked_activity
|
34
|
+
ON blocked_activity.pid = blocked_locks.pid
|
35
|
+
JOIN pg_catalog.pg_locks blocking_locks
|
36
|
+
ON blocking_locks.locktype = blocked_locks.locktype
|
37
|
+
AND blocking_locks.DATABASE IS NOT DISTINCT FROM blocked_locks.DATABASE
|
38
|
+
AND blocking_locks.relation IS NOT DISTINCT FROM blocked_locks.relation
|
39
|
+
AND blocking_locks.page IS NOT DISTINCT FROM blocked_locks.page
|
40
|
+
AND blocking_locks.tuple IS NOT DISTINCT FROM blocked_locks.tuple
|
41
|
+
AND blocking_locks.virtualxid IS NOT DISTINCT FROM blocked_locks.virtualxid
|
42
|
+
AND blocking_locks.transactionid IS NOT DISTINCT FROM blocked_locks.transactionid
|
43
|
+
AND blocking_locks.classid IS NOT DISTINCT FROM blocked_locks.classid
|
44
|
+
AND blocking_locks.objid IS NOT DISTINCT FROM blocked_locks.objid
|
45
|
+
AND blocking_locks.objsubid IS NOT DISTINCT FROM blocked_locks.objsubid
|
46
|
+
AND blocking_locks.pid != blocked_locks.pid
|
47
|
+
JOIN pg_catalog.pg_stat_activity blocking_activity
|
48
|
+
ON blocking_activity.pid = blocking_locks.pid
|
49
|
+
WHERE blocked_locks.pid = %d
|
50
|
+
SQL
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SafePgMigrations
|
4
|
+
module Helpers
|
5
|
+
module IndexHelper
|
6
|
+
def index_definition(table_name, column_name, **options)
|
7
|
+
index_definition, = add_index_options(table_name, column_name, **options)
|
8
|
+
index_definition
|
9
|
+
end
|
10
|
+
|
11
|
+
private
|
12
|
+
|
13
|
+
def index_valid?(index_name)
|
14
|
+
query_value <<~SQL.squish
|
15
|
+
SELECT indisvalid
|
16
|
+
FROM pg_index i
|
17
|
+
JOIN pg_class c
|
18
|
+
ON i.indexrelid = c.oid
|
19
|
+
WHERE c.relname = '#{index_name}';
|
20
|
+
SQL
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SafePgMigrations
|
4
|
+
module Helpers
|
5
|
+
module SatisfiedHelper
|
6
|
+
class << self
|
7
|
+
def satisfies_change_column_null_requirements?
|
8
|
+
satisfies_add_check_constraints? && SafePgMigrations.pg_version_num >= 120_000
|
9
|
+
end
|
10
|
+
|
11
|
+
def satisfies_add_check_constraints!
|
12
|
+
return if satisfies_add_check_constraints?
|
13
|
+
|
14
|
+
raise NotImplementedError, 'add_check_constraint is not supported in your ActiveRecord version'
|
15
|
+
end
|
16
|
+
|
17
|
+
def satisfies_add_check_constraints?
|
18
|
+
satisfied? '>=6.1.0'
|
19
|
+
end
|
20
|
+
|
21
|
+
def satisfies_add_column_update_rows_backfill?
|
22
|
+
satisfies_change_column_null_requirements?
|
23
|
+
end
|
24
|
+
|
25
|
+
def satisfied?(version)
|
26
|
+
Gem::Requirement.new(version).satisfied_by? Gem::Version.new(::ActiveRecord::VERSION::STRING)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -1,65 +1,62 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require_relative '../helpers/blocking_activity_formatter'
|
4
|
+
require_relative '../helpers/blocking_activity_selector'
|
5
|
+
|
3
6
|
module SafePgMigrations
|
4
|
-
module BlockingActivityLogger
|
5
|
-
|
6
|
-
|
7
|
-
blocked_locks.locktype
|
8
|
-
blocked_locks.mode
|
9
|
-
blocking_activity.pid
|
10
|
-
blocked_locks.transactionid
|
11
|
-
].freeze
|
12
|
-
|
13
|
-
VERBOSE_COLUMNS = %w[
|
14
|
-
blocking_activity.query
|
15
|
-
blocked_activity.xact_start
|
16
|
-
].freeze
|
7
|
+
module BlockingActivityLogger
|
8
|
+
include ::SafePgMigrations::Helpers::BlockingActivityFormatter
|
9
|
+
include ::SafePgMigrations::Helpers::BlockingActivitySelector
|
17
10
|
|
18
11
|
%i[
|
19
|
-
add_column
|
12
|
+
add_column
|
13
|
+
remove_column
|
14
|
+
add_foreign_key
|
15
|
+
remove_foreign_key
|
16
|
+
change_column_default
|
17
|
+
change_column_null
|
18
|
+
create_table
|
20
19
|
].each do |method|
|
21
20
|
define_method method do |*args, &block|
|
22
|
-
|
21
|
+
log_blocking_queries_after_lock { super(*args, &block) }
|
23
22
|
end
|
24
23
|
ruby2_keywords method
|
25
24
|
end
|
26
25
|
|
27
|
-
|
26
|
+
%i[add_index remove_index].each do |method|
|
27
|
+
define_method method do |*args, **options, &block|
|
28
|
+
return super(*args, **options, &block) if options[:algorithm] != :concurrently
|
28
29
|
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
<<~SQL.squish
|
33
|
-
SELECT #{columns.join(', ')}
|
34
|
-
FROM pg_catalog.pg_locks blocked_locks
|
35
|
-
JOIN pg_catalog.pg_stat_activity blocked_activity
|
36
|
-
ON blocked_activity.pid = blocked_locks.pid
|
37
|
-
JOIN pg_catalog.pg_locks blocking_locks
|
38
|
-
ON blocking_locks.locktype = blocked_locks.locktype
|
39
|
-
AND blocking_locks.DATABASE IS NOT DISTINCT FROM blocked_locks.DATABASE
|
40
|
-
AND blocking_locks.relation IS NOT DISTINCT FROM blocked_locks.relation
|
41
|
-
AND blocking_locks.page IS NOT DISTINCT FROM blocked_locks.page
|
42
|
-
AND blocking_locks.tuple IS NOT DISTINCT FROM blocked_locks.tuple
|
43
|
-
AND blocking_locks.virtualxid IS NOT DISTINCT FROM blocked_locks.virtualxid
|
44
|
-
AND blocking_locks.transactionid IS NOT DISTINCT FROM blocked_locks.transactionid
|
45
|
-
AND blocking_locks.classid IS NOT DISTINCT FROM blocked_locks.classid
|
46
|
-
AND blocking_locks.objid IS NOT DISTINCT FROM blocked_locks.objid
|
47
|
-
AND blocking_locks.objsubid IS NOT DISTINCT FROM blocked_locks.objsubid
|
48
|
-
AND blocking_locks.pid != blocked_locks.pid
|
49
|
-
JOIN pg_catalog.pg_stat_activity blocking_activity
|
50
|
-
ON blocking_activity.pid = blocking_locks.pid
|
51
|
-
WHERE blocked_locks.pid = %d
|
52
|
-
SQL
|
30
|
+
log_blocking_queries_loop { super(*args, **options, &block) }
|
31
|
+
end
|
53
32
|
end
|
54
33
|
|
55
|
-
|
56
|
-
|
57
|
-
|
34
|
+
private
|
35
|
+
|
36
|
+
def log_blocking_queries_loop
|
37
|
+
blocking_queries_retriever_thread =
|
38
|
+
Thread.new do
|
39
|
+
loop do
|
40
|
+
sleep SafePgMigrations.config.retry_delay
|
41
|
+
|
42
|
+
log_queries SafePgMigrations.alternate_connection.query(
|
43
|
+
select_blocking_queries_sql % raw_connection.backend_pid
|
44
|
+
)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
yield
|
49
|
+
|
50
|
+
blocking_queries_retriever_thread.kill
|
51
|
+
end
|
58
52
|
|
53
|
+
def log_blocking_queries_after_lock
|
59
54
|
blocking_queries_retriever_thread =
|
60
55
|
Thread.new do
|
61
56
|
sleep delay_before_logging
|
62
|
-
SafePgMigrations.alternate_connection.query(
|
57
|
+
SafePgMigrations.alternate_connection.query(
|
58
|
+
select_blocking_queries_sql % raw_connection.backend_pid
|
59
|
+
)
|
63
60
|
end
|
64
61
|
|
65
62
|
yield
|
@@ -71,51 +68,25 @@ module SafePgMigrations
|
|
71
68
|
begin
|
72
69
|
blocking_queries_retriever_thread.value
|
73
70
|
rescue StandardError => e
|
74
|
-
SafePgMigrations.say(
|
71
|
+
SafePgMigrations.say(
|
72
|
+
"Error while retrieving blocking queries: #{e}",
|
73
|
+
true
|
74
|
+
)
|
75
75
|
nil
|
76
76
|
end
|
77
77
|
|
78
|
-
|
79
|
-
|
80
|
-
if queries.empty?
|
81
|
-
SafePgMigrations.say 'Could not find any blocking query.', true
|
82
|
-
else
|
83
|
-
SafePgMigrations.say(
|
84
|
-
"Statement was being blocked by the following #{'query'.pluralize(queries.size)}:", true
|
85
|
-
)
|
86
|
-
SafePgMigrations.say '', true
|
87
|
-
output_blocking_queries(queries)
|
88
|
-
SafePgMigrations.say(
|
89
|
-
'Beware, some of those queries might run in a transaction. In this case the locking query might be '\
|
90
|
-
'located elsewhere in the transaction',
|
91
|
-
true
|
92
|
-
)
|
93
|
-
SafePgMigrations.say '', true
|
94
|
-
end
|
78
|
+
log_queries queries unless queries.nil?
|
95
79
|
|
96
80
|
raise
|
97
81
|
end
|
98
82
|
|
99
|
-
def
|
100
|
-
|
101
|
-
|
102
|
-
else
|
103
|
-
queries.each do |start_time, locktype, mode, pid, transactionid|
|
104
|
-
SafePgMigrations.say(
|
105
|
-
"#{format_start_time(start_time)}: lock type: #{locktype || 'null'}, " \
|
106
|
-
"lock mode: #{mode || 'null'}, " \
|
107
|
-
"lock pid: #{pid || 'null'}, " \
|
108
|
-
"lock transactionid: #{transactionid || 'null'}",
|
109
|
-
true
|
110
|
-
)
|
111
|
-
end
|
112
|
-
end
|
83
|
+
def delay_before_logging
|
84
|
+
SafePgMigrations.config.safe_timeout -
|
85
|
+
SafePgMigrations.config.blocking_activity_logger_margin
|
113
86
|
end
|
114
87
|
|
115
|
-
def
|
116
|
-
|
117
|
-
duration = (reference_time - start_time).round
|
118
|
-
"transaction started #{duration} #{'second'.pluralize(duration)} ago"
|
88
|
+
def delay_before_retry
|
89
|
+
SafePgMigrations.config.blocking_activity_logger_margin + SafePgMigrations.config.retry_delay
|
119
90
|
end
|
120
91
|
end
|
121
92
|
end
|
@@ -2,6 +2,8 @@
|
|
2
2
|
|
3
3
|
module SafePgMigrations
|
4
4
|
module IdempotentStatements
|
5
|
+
include SafePgMigrations::Helpers::IndexHelper
|
6
|
+
|
5
7
|
ruby2_keywords def add_index(table_name, column_name, *args)
|
6
8
|
options = args.last.is_a?(Hash) ? args.last : {}
|
7
9
|
|
@@ -53,6 +55,16 @@ module SafePgMigrations
|
|
53
55
|
)
|
54
56
|
end
|
55
57
|
|
58
|
+
def remove_foreign_key(from_table, to_table = nil, **options)
|
59
|
+
return super if foreign_key_exists?(from_table, to_table, **options)
|
60
|
+
|
61
|
+
reference_name = to_table || options[:to_table] || options[:column] || options[:name]
|
62
|
+
SafePgMigrations.say(
|
63
|
+
"/!\\ Foreign key '#{from_table}' -> '#{reference_name}' does not exist. Skipping statement.",
|
64
|
+
true
|
65
|
+
)
|
66
|
+
end
|
67
|
+
|
56
68
|
ruby2_keywords def create_table(table_name, *args)
|
57
69
|
options = args.last.is_a?(Hash) ? args.last : {}
|
58
70
|
return super if options[:force] || !table_exists?(table_name)
|
@@ -70,23 +82,51 @@ module SafePgMigrations
|
|
70
82
|
end
|
71
83
|
end
|
72
84
|
|
73
|
-
|
85
|
+
def add_check_constraint(table_name, expression, **options)
|
86
|
+
constraint_definition = check_constraint_for table_name,
|
87
|
+
**check_constraint_options(table_name, expression, options)
|
74
88
|
|
75
|
-
|
76
|
-
|
77
|
-
|
89
|
+
return super if constraint_definition.nil?
|
90
|
+
|
91
|
+
SafePgMigrations.say "/!\\ Constraint '#{constraint_definition.name}' already exists. Skipping statement.", true
|
78
92
|
end
|
79
93
|
|
80
|
-
|
94
|
+
def change_column_null(table_name, column_name, null, *)
|
95
|
+
column = column_for(table_name, column_name)
|
96
|
+
|
97
|
+
return super if column.null != null
|
81
98
|
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
99
|
+
SafePgMigrations.say(
|
100
|
+
"/!\\ Column '#{table_name}.#{column.name}' is already set to 'null: #{null}'. Skipping statement.",
|
101
|
+
true
|
102
|
+
)
|
103
|
+
end
|
104
|
+
|
105
|
+
def validate_check_constraint(table_name, **options)
|
106
|
+
constraint_definition = check_constraint_for!(table_name, **options)
|
107
|
+
|
108
|
+
return super unless constraint_definition.validated?
|
109
|
+
|
110
|
+
SafePgMigrations.say "/!\\ Constraint '#{constraint_definition.name}' already validated. Skipping statement.",
|
111
|
+
true
|
112
|
+
end
|
113
|
+
|
114
|
+
def change_column_default(table_name, column_name, default_or_changes)
|
115
|
+
column = column_for(table_name, column_name)
|
116
|
+
|
117
|
+
previous_alter_statement = change_column_default_for_alter(table_name, column_name, column.default)
|
118
|
+
new_alter_statement = change_column_default_for_alter(table_name, column_name, default_or_changes)
|
119
|
+
|
120
|
+
# NOTE: PG change_column_default is already idempotent.
|
121
|
+
# We try to detect it because it still takes an ACCESS EXCLUSIVE lock
|
122
|
+
|
123
|
+
return super if new_alter_statement != previous_alter_statement
|
124
|
+
|
125
|
+
SafePgMigrations.say(
|
126
|
+
"/!\\ Column '#{table_name}.#{column.name}' is already set to 'default: #{column.default}'. " \
|
127
|
+
'Skipping statement.',
|
128
|
+
true
|
129
|
+
)
|
90
130
|
end
|
91
131
|
end
|
92
132
|
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SafePgMigrations
|
4
|
+
module StatementInsurer
|
5
|
+
module AddColumn
|
6
|
+
ruby2_keywords def add_column(table_name, column_name, type, *args)
|
7
|
+
options = args.last.is_a?(Hash) && args.last
|
8
|
+
options ||= {}
|
9
|
+
|
10
|
+
if should_keep_default_implementation?(**options)
|
11
|
+
with_setting(:statement_timeout, SafePgMigrations.config.pg_statement_timeout) { return super }
|
12
|
+
end
|
13
|
+
|
14
|
+
default = options.delete(:default)
|
15
|
+
null = options.delete(:null)
|
16
|
+
|
17
|
+
with_setting(:statement_timeout, SafePgMigrations.config.pg_statement_timeout) do
|
18
|
+
SafePgMigrations.say_method_call(:add_column, table_name, column_name, type, options)
|
19
|
+
super table_name, column_name, type, **options
|
20
|
+
end
|
21
|
+
|
22
|
+
SafePgMigrations.say_method_call(:change_column_default, table_name, column_name, default)
|
23
|
+
change_column_default(table_name, column_name, default)
|
24
|
+
|
25
|
+
SafePgMigrations.say_method_call(:backfill_column_default, table_name, column_name)
|
26
|
+
without_statement_timeout do
|
27
|
+
backfill_column_default(table_name, column_name)
|
28
|
+
end
|
29
|
+
|
30
|
+
change_column_null(table_name, column_name, null) if null == false
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
|
35
|
+
def should_keep_default_implementation?(default: nil, default_value_backfill: :auto, **)
|
36
|
+
default_value_backfill != :update_in_batches || !default ||
|
37
|
+
!SafePgMigrations::Helpers::SatisfiedHelper.satisfies_add_column_update_rows_backfill?
|
38
|
+
end
|
39
|
+
|
40
|
+
def backfill_column_default(table_name, column_name)
|
41
|
+
model = Class.new(ActiveRecord::Base) { self.table_name = table_name }
|
42
|
+
quoted_column_name = quote_column_name(column_name)
|
43
|
+
|
44
|
+
SafePgMigrations::Helpers::BatchOver.new(model).each_batch do |batch|
|
45
|
+
batch
|
46
|
+
.update_all("#{quoted_column_name} = DEFAULT")
|
47
|
+
sleep SafePgMigrations.config.backfill_pause
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -2,47 +2,41 @@
|
|
2
2
|
|
3
3
|
module SafePgMigrations
|
4
4
|
module StatementInsurer
|
5
|
-
|
5
|
+
include AddColumn
|
6
6
|
|
7
|
-
%i[
|
7
|
+
%i[change_column].each do |method|
|
8
8
|
define_method method do |*args, &block|
|
9
|
-
with_setting(:statement_timeout, SafePgMigrations.config.
|
9
|
+
with_setting(:statement_timeout, SafePgMigrations.config.pg_statement_timeout) { super(*args, &block) }
|
10
10
|
end
|
11
11
|
ruby2_keywords method
|
12
12
|
end
|
13
13
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
default = options.delete(:default)
|
19
|
-
null = options.delete(:null)
|
20
|
-
|
21
|
-
if !default.nil? || null == false
|
22
|
-
SafePgMigrations.say_method_call(:add_column, table_name, column_name, type, options)
|
14
|
+
def validate_check_constraint(table_name, **options)
|
15
|
+
without_statement_timeout do
|
16
|
+
super
|
23
17
|
end
|
18
|
+
end
|
24
19
|
|
25
|
-
|
20
|
+
def add_check_constraint(table_name, expression, **options)
|
21
|
+
Helpers::SatisfiedHelper.satisfies_add_check_constraints!
|
22
|
+
return unless supports_check_constraints?
|
26
23
|
|
27
|
-
|
28
|
-
SafePgMigrations.say_method_call(:change_column_default, table_name, column_name, default)
|
29
|
-
change_column_default(table_name, column_name, default)
|
24
|
+
options = check_constraint_options(table_name, expression, options)
|
30
25
|
|
31
|
-
|
32
|
-
|
33
|
-
end
|
26
|
+
SafePgMigrations.say_method_call :add_check_constraint, table_name, expression, **options, validate: false
|
27
|
+
super table_name, expression, **options, validate: false
|
34
28
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
29
|
+
return unless options.fetch(:validate, true)
|
30
|
+
|
31
|
+
SafePgMigrations.say_method_call :validate_check_constraint, table_name, name: options[:name]
|
32
|
+
validate_check_constraint table_name, name: options[:name]
|
39
33
|
end
|
40
34
|
|
41
35
|
ruby2_keywords def add_foreign_key(from_table, to_table, *args)
|
42
36
|
options = args.last.is_a?(Hash) ? args.last : {}
|
43
|
-
validate_present = options.key?
|
37
|
+
validate_present = options.key?(:validate)
|
44
38
|
options[:validate] = false unless validate_present
|
45
|
-
with_setting(:statement_timeout, SafePgMigrations.config.
|
39
|
+
with_setting(:statement_timeout, SafePgMigrations.config.pg_statement_timeout) do
|
46
40
|
super(from_table, to_table, **options)
|
47
41
|
end
|
48
42
|
|
@@ -53,7 +47,7 @@ module SafePgMigrations
|
|
53
47
|
end
|
54
48
|
|
55
49
|
ruby2_keywords def create_table(*)
|
56
|
-
with_setting(:statement_timeout, SafePgMigrations.config.
|
50
|
+
with_setting(:statement_timeout, SafePgMigrations.config.pg_statement_timeout) do
|
57
51
|
super do |td|
|
58
52
|
yield td if block_given?
|
59
53
|
td.indexes.map! do |key, index_options|
|
@@ -64,7 +58,9 @@ module SafePgMigrations
|
|
64
58
|
end
|
65
59
|
end
|
66
60
|
|
67
|
-
def add_index(table_name, column_name,
|
61
|
+
ruby2_keywords def add_index(table_name, column_name, *args_options)
|
62
|
+
options = args_options.last.is_a?(Hash) ? args_options.last : {}
|
63
|
+
|
68
64
|
if options[:algorithm] == :default
|
69
65
|
options.delete :algorithm
|
70
66
|
else
|
@@ -84,13 +80,20 @@ module SafePgMigrations
|
|
84
80
|
without_timeout { super(table_name, **options) }
|
85
81
|
end
|
86
82
|
|
87
|
-
def
|
88
|
-
|
89
|
-
|
83
|
+
def change_column_null(table_name, column_name, null, default = nil)
|
84
|
+
if default || null || !Helpers::SatisfiedHelper.satisfies_change_column_null_requirements?
|
85
|
+
with_setting(:statement_timeout, SafePgMigrations.config.pg_statement_timeout) { return super }
|
86
|
+
end
|
90
87
|
|
91
|
-
|
92
|
-
|
88
|
+
add_check_constraint table_name, "#{column_name} IS NOT NULL"
|
89
|
+
|
90
|
+
SafePgMigrations.say_method_call :change_column_null, table_name, column_name, false
|
91
|
+
with_setting(:statement_timeout, SafePgMigrations.config.pg_statement_timeout) do
|
92
|
+
super table_name, column_name, false
|
93
93
|
end
|
94
|
+
|
95
|
+
SafePgMigrations.say_method_call :remove_check_constraint, table_name, "#{column_name} IS NOT NULL"
|
96
|
+
remove_check_constraint table_name, "#{column_name} IS NOT NULL"
|
94
97
|
end
|
95
98
|
|
96
99
|
def with_setting(key, value)
|
@@ -109,16 +112,16 @@ module SafePgMigrations
|
|
109
112
|
end
|
110
113
|
end
|
111
114
|
|
112
|
-
def without_statement_timeout
|
113
|
-
with_setting(:statement_timeout, 0)
|
115
|
+
def without_statement_timeout(&block)
|
116
|
+
with_setting(:statement_timeout, 0, &block)
|
114
117
|
end
|
115
118
|
|
116
|
-
def without_lock_timeout
|
117
|
-
with_setting(:lock_timeout, 0)
|
119
|
+
def without_lock_timeout(&block)
|
120
|
+
with_setting(:lock_timeout, 0, &block)
|
118
121
|
end
|
119
122
|
|
120
|
-
def without_timeout
|
121
|
-
without_statement_timeout { without_lock_timeout
|
123
|
+
def without_timeout(&block)
|
124
|
+
without_statement_timeout { without_lock_timeout(&block) }
|
122
125
|
end
|
123
126
|
end
|
124
127
|
end
|
@@ -3,7 +3,7 @@
|
|
3
3
|
module SafePgMigrations
|
4
4
|
module StatementRetrier
|
5
5
|
RETRIABLE_SCHEMA_STATEMENTS = %i[
|
6
|
-
add_column add_foreign_key remove_foreign_key change_column_default change_column_null remove_column
|
6
|
+
add_column add_foreign_key remove_foreign_key change_column_default change_column_null remove_column drop_table
|
7
7
|
].freeze
|
8
8
|
|
9
9
|
RETRIABLE_SCHEMA_STATEMENTS.each do |method|
|
@@ -29,6 +29,14 @@ module SafePgMigrations
|
|
29
29
|
super
|
30
30
|
end
|
31
31
|
|
32
|
+
ruby2_keywords def add_check_constraint(*args)
|
33
|
+
options = args.last.is_a?(Hash) ? args.last : {}
|
34
|
+
if options[:validate] == false
|
35
|
+
UselessStatementsLogger.warn_useless '`validate: :false`', 'https://github.com/doctolib/safe-pg-migrations#safe_add_check_constraint'
|
36
|
+
end
|
37
|
+
super
|
38
|
+
end
|
39
|
+
|
32
40
|
def warn_for_index(**options)
|
33
41
|
return unless options[:algorithm] == :concurrently
|
34
42
|
|
@@ -4,7 +4,7 @@ module SafePgMigrations
|
|
4
4
|
class VerboseSqlLogger
|
5
5
|
def setup
|
6
6
|
@activerecord_logger_was = ActiveRecord::Base.logger
|
7
|
-
@verbose_query_logs_was =
|
7
|
+
@verbose_query_logs_was = Polyfills::VerboseQueryLogsPolyfill.verbose_query_logs
|
8
8
|
@colorize_logging_was = ActiveRecord::LogSubscriber.colorize_logging
|
9
9
|
|
10
10
|
disable_marginalia if defined?(Marginalia)
|
@@ -13,12 +13,12 @@ module SafePgMigrations
|
|
13
13
|
ActiveRecord::Base.logger = stdout_logger
|
14
14
|
ActiveRecord::LogSubscriber.colorize_logging = colorize_logging?
|
15
15
|
# Do not output caller method, we know it is coming from the migration
|
16
|
-
|
16
|
+
Polyfills::VerboseQueryLogsPolyfill.verbose_query_logs = false
|
17
17
|
self
|
18
18
|
end
|
19
19
|
|
20
20
|
def teardown
|
21
|
-
|
21
|
+
Polyfills::VerboseQueryLogsPolyfill.verbose_query_logs = @verbose_query_logs_was
|
22
22
|
ActiveRecord::LogSubscriber.colorize_logging = @colorize_logging_was
|
23
23
|
ActiveRecord::Base.logger = @activerecord_logger_was
|
24
24
|
enable_marginalia if defined?(Marginalia)
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SafePgMigrations
|
4
|
+
module Polyfills
|
5
|
+
module IndexDefinitionPolyfill
|
6
|
+
include Helpers::SatisfiedHelper
|
7
|
+
|
8
|
+
protected
|
9
|
+
|
10
|
+
IndexDefinition = Struct.new(:table, :name)
|
11
|
+
|
12
|
+
def index_definition(table_name, column_name, **options)
|
13
|
+
return super(table_name, column_name, **options) if Helpers::SatisfiedHelper.satisfied? '>=6.1.0'
|
14
|
+
|
15
|
+
index_name = options.key?(:name) ? options[:name].to_s : index_name(table_name, index_column_names(column_name))
|
16
|
+
validate_index_length!(table_name, index_name, options.fetch(:internal, false))
|
17
|
+
|
18
|
+
IndexDefinition.new(table_name, index_name)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SafePgMigrations
|
4
|
+
module Polyfills
|
5
|
+
module VerboseQueryLogsPolyfill
|
6
|
+
class << self
|
7
|
+
def verbose_query_logs
|
8
|
+
return ActiveRecord.verbose_query_logs if Helpers::SatisfiedHelper.satisfied? '>=7.0.0'
|
9
|
+
|
10
|
+
ActiveRecord::Base.verbose_query_logs
|
11
|
+
end
|
12
|
+
|
13
|
+
def verbose_query_logs=(value)
|
14
|
+
if Helpers::SatisfiedHelper.satisfied? '>=7.0.0'
|
15
|
+
ActiveRecord.verbose_query_logs = value
|
16
|
+
return
|
17
|
+
end
|
18
|
+
|
19
|
+
ActiveRecord::Base.verbose_query_logs = value
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -7,6 +7,7 @@ module SafePgMigrations
|
|
7
7
|
initializer 'safe_pg_migrations.insert_into_active_record' do
|
8
8
|
ActiveSupport.on_load :active_record do
|
9
9
|
ActiveRecord::Migration.prepend(SafePgMigrations::Migration)
|
10
|
+
ActiveRecord::Migration.singleton_class.prepend(SafePgMigrations::Migration::ClassMethods)
|
10
11
|
end
|
11
12
|
end
|
12
13
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: safe-pg-migrations
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 2.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Matthieu Prat
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date:
|
13
|
+
date: 2023-06-29 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: activerecord
|
@@ -18,42 +18,28 @@ dependencies:
|
|
18
18
|
requirements:
|
19
19
|
- - ">="
|
20
20
|
- !ruby/object:Gem::Version
|
21
|
-
version: '
|
21
|
+
version: '6.0'
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
24
|
version_requirements: !ruby/object:Gem::Requirement
|
25
25
|
requirements:
|
26
26
|
- - ">="
|
27
27
|
- !ruby/object:Gem::Version
|
28
|
-
version: '
|
28
|
+
version: '6.0'
|
29
29
|
- !ruby/object:Gem::Dependency
|
30
30
|
name: activesupport
|
31
31
|
requirement: !ruby/object:Gem::Requirement
|
32
32
|
requirements:
|
33
33
|
- - ">="
|
34
34
|
- !ruby/object:Gem::Version
|
35
|
-
version: '
|
35
|
+
version: '6.0'
|
36
36
|
type: :runtime
|
37
37
|
prerelease: false
|
38
38
|
version_requirements: !ruby/object:Gem::Requirement
|
39
39
|
requirements:
|
40
40
|
- - ">="
|
41
41
|
- !ruby/object:Gem::Version
|
42
|
-
version: '
|
43
|
-
- !ruby/object:Gem::Dependency
|
44
|
-
name: ruby2_keywords
|
45
|
-
requirement: !ruby/object:Gem::Requirement
|
46
|
-
requirements:
|
47
|
-
- - ">="
|
48
|
-
- !ruby/object:Gem::Version
|
49
|
-
version: 0.0.4
|
50
|
-
type: :runtime
|
51
|
-
prerelease: false
|
52
|
-
version_requirements: !ruby/object:Gem::Requirement
|
53
|
-
requirements:
|
54
|
-
- - ">="
|
55
|
-
- !ruby/object:Gem::Version
|
56
|
-
version: 0.0.4
|
42
|
+
version: '6.0'
|
57
43
|
description: Make your PG migrations safe.
|
58
44
|
email:
|
59
45
|
executables: []
|
@@ -65,13 +51,20 @@ files:
|
|
65
51
|
- lib/safe-pg-migrations.rb
|
66
52
|
- lib/safe-pg-migrations/base.rb
|
67
53
|
- lib/safe-pg-migrations/configuration.rb
|
54
|
+
- lib/safe-pg-migrations/helpers/batch_over.rb
|
55
|
+
- lib/safe-pg-migrations/helpers/blocking_activity_formatter.rb
|
56
|
+
- lib/safe-pg-migrations/helpers/blocking_activity_selector.rb
|
57
|
+
- lib/safe-pg-migrations/helpers/index_helper.rb
|
58
|
+
- lib/safe-pg-migrations/helpers/satisfied_helper.rb
|
68
59
|
- lib/safe-pg-migrations/plugins/blocking_activity_logger.rb
|
69
60
|
- lib/safe-pg-migrations/plugins/idempotent_statements.rb
|
70
|
-
- lib/safe-pg-migrations/plugins/legacy_active_record_support.rb
|
71
61
|
- lib/safe-pg-migrations/plugins/statement_insurer.rb
|
62
|
+
- lib/safe-pg-migrations/plugins/statement_insurer/add_column.rb
|
72
63
|
- lib/safe-pg-migrations/plugins/statement_retrier.rb
|
73
64
|
- lib/safe-pg-migrations/plugins/useless_statements_logger.rb
|
74
65
|
- lib/safe-pg-migrations/plugins/verbose_sql_logger.rb
|
66
|
+
- lib/safe-pg-migrations/polyfills/index_definition_polyfill.rb
|
67
|
+
- lib/safe-pg-migrations/polyfills/verbose_query_logs_polyfill.rb
|
75
68
|
- lib/safe-pg-migrations/railtie.rb
|
76
69
|
- lib/safe-pg-migrations/version.rb
|
77
70
|
homepage: https://github.com/doctolib/safe-pg-migrations
|
@@ -83,6 +76,7 @@ metadata:
|
|
83
76
|
mailing_list_uri: https://doctolib.engineering/engineering-news-ruby-rails-react
|
84
77
|
source_code_uri: https://github.com/doctolib/safe-pg-migrations
|
85
78
|
contributors_uri: https://github.com/doctolib/safe-pg-migrations/graphs/contributors
|
79
|
+
rubygems_mfa_required: 'true'
|
86
80
|
post_install_message:
|
87
81
|
rdoc_options: []
|
88
82
|
require_paths:
|
@@ -91,18 +85,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
91
85
|
requirements:
|
92
86
|
- - ">="
|
93
87
|
- !ruby/object:Gem::Version
|
94
|
-
version: '2.
|
95
|
-
- - "<"
|
96
|
-
- !ruby/object:Gem::Version
|
97
|
-
version: '4'
|
88
|
+
version: '2.7'
|
98
89
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
99
90
|
requirements:
|
100
91
|
- - ">="
|
101
92
|
- !ruby/object:Gem::Version
|
102
93
|
version: '0'
|
103
94
|
requirements: []
|
104
|
-
|
105
|
-
rubygems_version: 2.7.3
|
95
|
+
rubygems_version: 3.3.7
|
106
96
|
signing_key:
|
107
97
|
specification_version: 4
|
108
98
|
summary: Make your PG migrations safe.
|
@@ -1,38 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module SafePgMigrations
|
4
|
-
module LegacyActiveRecordSupport
|
5
|
-
ruby2_keywords def validate_foreign_key(*args)
|
6
|
-
return super(*args) if satisfied? '>=6.0.0'
|
7
|
-
|
8
|
-
from_table, to_table, options = args
|
9
|
-
super(from_table, to_table || options)
|
10
|
-
end
|
11
|
-
|
12
|
-
ruby2_keywords def foreign_key_exists?(*args)
|
13
|
-
return super(*args) if satisfied? '>=6.0.0'
|
14
|
-
|
15
|
-
from_table, to_table, options = args
|
16
|
-
super(from_table, to_table || options)
|
17
|
-
end
|
18
|
-
|
19
|
-
protected
|
20
|
-
|
21
|
-
IndexDefinition = Struct.new(:table, :name)
|
22
|
-
|
23
|
-
def index_definition(table_name, column_name, **options)
|
24
|
-
return super(table_name, column_name, **options) if satisfied? '>=6.1.0'
|
25
|
-
|
26
|
-
index_name = options.key?(:name) ? options[:name].to_s : index_name(table_name, index_column_names(column_name))
|
27
|
-
validate_index_length!(table_name, index_name, options.fetch(:internal, false))
|
28
|
-
|
29
|
-
IndexDefinition.new(table_name, index_name)
|
30
|
-
end
|
31
|
-
|
32
|
-
private
|
33
|
-
|
34
|
-
def satisfied?(version)
|
35
|
-
Gem::Requirement.new(version).satisfied_by? Gem::Version.new(::ActiveRecord::VERSION::STRING)
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|