safe-pg-migrations 1.4.2 → 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +23 -36
- data/lib/safe-pg-migrations/base.rb +18 -11
- data/lib/safe-pg-migrations/configuration.rb +2 -6
- data/lib/safe-pg-migrations/helpers/blocking_activity_formatter.rb +59 -0
- data/lib/safe-pg-migrations/helpers/blocking_activity_selector.rb +54 -0
- data/lib/safe-pg-migrations/plugins/blocking_activity_logger.rb +51 -80
- data/lib/safe-pg-migrations/plugins/idempotent_statements.rb +10 -0
- data/lib/safe-pg-migrations/plugins/statement_insurer.rb +9 -45
- data/lib/safe-pg-migrations/plugins/verbose_sql_logger.rb +3 -3
- data/lib/safe-pg-migrations/polyfills/index_definition_polyfill.rb +22 -0
- data/lib/safe-pg-migrations/polyfills/satisfied_helper.rb +13 -0
- data/lib/safe-pg-migrations/polyfills/verbose_query_logs_polyfill.rb +26 -0
- data/lib/safe-pg-migrations/railtie.rb +1 -0
- data/lib/safe-pg-migrations/version.rb +1 -1
- metadata +14 -27
- data/lib/safe-pg-migrations/plugins/legacy_active_record_support.rb +0 -38
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 878eca03499c2fa44ef8e046f5828b2b0978e692ade4440f9489a29889a99b49
|
4
|
+
data.tar.gz: 88a9aa0adf58a3a716565d8a3482839ffd546570ab7238f67daecb0c2396a152
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 15cd514d22cf2b5f182faf16a50a4775fa528754357eeeaa0c81f2f93bf25d12375ae2c7742e8e762addd01ae3b5aeeedb09c758e9e6da063f4b0beca8544c45
|
7
|
+
data.tar.gz: d930ac0e741025ea05743a2ad8f6da979932f67cd9e93e13c64b22581bb073b164212144d152f9a0729a93e49098cb9279caba57a91b746502ccb259cb18bbcb
|
data/README.md
CHANGED
@@ -6,9 +6,9 @@ ActiveRecord migrations for Postgres made safe.
|
|
6
6
|
|
7
7
|
## Requirements
|
8
8
|
|
9
|
-
- Ruby 2.
|
10
|
-
- Rails
|
11
|
-
- PostgreSQL
|
9
|
+
- Ruby 2.7+
|
10
|
+
- Rails 6.0+
|
11
|
+
- PostgreSQL 11.7+
|
12
12
|
|
13
13
|
## Usage
|
14
14
|
|
@@ -25,43 +25,46 @@ gem 'safe-pg-migrations'
|
|
25
25
|
Consider the following migration:
|
26
26
|
|
27
27
|
```rb
|
28
|
-
class
|
28
|
+
class AddPatientRefToAppointments < ActiveRecord::Migration[6.0]
|
29
29
|
def change
|
30
|
-
|
30
|
+
add_reference :appointments, :patient
|
31
31
|
end
|
32
32
|
end
|
33
33
|
```
|
34
34
|
|
35
|
-
If the `users` table is large, running this migration
|
35
|
+
If the `users` table is large, running this migration will likely cause downtime. **Safe PG Migrations** hooks into Active Record so that the following gets executed instead:
|
36
36
|
|
37
37
|
```rb
|
38
|
-
class
|
38
|
+
class AddPatientRefToAppointments < ActiveRecord::Migration[6.0]
|
39
39
|
# Do not wrap the migration in a transaction so that locks are held for a shorter time.
|
40
40
|
disable_ddl_transaction!
|
41
41
|
|
42
42
|
def change
|
43
43
|
# Lower Postgres' lock timeout to avoid statement queueing. Acts like a seatbelt.
|
44
|
-
execute
|
44
|
+
execute("SET lock_timeout TO '5s'")
|
45
45
|
|
46
|
-
#
|
47
|
-
|
46
|
+
# Lower Postgres' statement timeout to avoid too long transactions. Acts like a seatbelt.
|
47
|
+
execute("SET statement_timeout TO '5s'")
|
48
|
+
add_column :appointments, :patient_id, :bigint
|
48
49
|
|
49
|
-
#
|
50
|
-
|
50
|
+
# add_index using the concurrent algorithm, to avoid locking the tables
|
51
|
+
add_index :appointments, :patient_id, algorithm: :concurrently
|
51
52
|
|
52
|
-
#
|
53
|
-
|
53
|
+
# add_foreign_key without validation, to avoid locking the table for too long
|
54
|
+
execute("SET statement_timeout TO '5s'")
|
55
|
+
add_foreign_key :appointments, :patients, validate: false
|
54
56
|
|
55
|
-
|
56
|
-
|
57
|
-
# the
|
58
|
-
|
59
|
-
|
57
|
+
execute("SET statement_timeout TO '0'")
|
58
|
+
|
59
|
+
# validate the foreign key separately, it avoids taking a lock on the entire tables
|
60
|
+
validate_foreign_key :appointments, :patients
|
61
|
+
|
62
|
+
# we also need to set timeouts to their initial values if needed
|
60
63
|
end
|
61
64
|
end
|
62
65
|
```
|
63
66
|
|
64
|
-
Under the hood, **Safe PG Migrations** patches `ActiveRecord::Migration` and extends `ActiveRecord::Base.connection` to make potentially dangerous methods—like `
|
67
|
+
Under the hood, **Safe PG Migrations** patches `ActiveRecord::Migration` and extends `ActiveRecord::Base.connection` to make potentially dangerous methods—like `add_reference`—safe.
|
65
68
|
|
66
69
|
## Motivation
|
67
70
|
|
@@ -108,22 +111,6 @@ add_column :users, :created_at, default: 'clock_timestamp()'
|
|
108
111
|
```
|
109
112
|
PG will still needs to update every row of the table, and will most likely statement timeout for big table. In this case, your best bet is to add the column without default, set the default, and backfill existing rows.
|
110
113
|
|
111
|
-
<blockquote>
|
112
|
-
|
113
|
-
**Note: Pre-postgres 11**
|
114
|
-
Adding a column with a default value and a not-null constraint is [dangerous](https://wework.github.io/data/2015/11/05/add-columns-with-default-values-to-large-tables-in-rails-postgres/).
|
115
|
-
|
116
|
-
**Safe PG Migrations** makes it safe by:
|
117
|
-
|
118
|
-
1. Adding the column without the default value and the not null constraint,
|
119
|
-
2. Then set the default value on the column,
|
120
|
-
3. Then backfilling the column,
|
121
|
-
4. And then adding the not null constraint with a short statement timeout.
|
122
|
-
|
123
|
-
Note: the addition of the not null constraint may timeout. In that case, you may want to add the not-null constraint as initially not valid and validate it in a separate statement. See [Adding a not-null constraint on Postgres with minimal locking](https://medium.com/doctolib-engineering/adding-a-not-null-constraint-on-pg-faster-with-minimal-locking-38b2c00c4d1c).
|
124
|
-
|
125
|
-
</blockquote>
|
126
|
-
|
127
114
|
</details>
|
128
115
|
|
129
116
|
<details><summary id="safe_add_remove_index">Safe <code>add_index</code> and <code>remove_index</code></summary>
|
@@ -1,6 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require 'ruby2_keywords'
|
4
3
|
require 'safe-pg-migrations/configuration'
|
5
4
|
require 'safe-pg-migrations/plugins/verbose_sql_logger'
|
6
5
|
require 'safe-pg-migrations/plugins/blocking_activity_logger'
|
@@ -8,7 +7,9 @@ require 'safe-pg-migrations/plugins/statement_insurer'
|
|
8
7
|
require 'safe-pg-migrations/plugins/statement_retrier'
|
9
8
|
require 'safe-pg-migrations/plugins/idempotent_statements'
|
10
9
|
require 'safe-pg-migrations/plugins/useless_statements_logger'
|
11
|
-
require 'safe-pg-migrations/
|
10
|
+
require 'safe-pg-migrations/polyfills/satisfied_helper'
|
11
|
+
require 'safe-pg-migrations/polyfills/index_definition_polyfill'
|
12
|
+
require 'safe-pg-migrations/polyfills/verbose_query_logs_polyfill'
|
12
13
|
|
13
14
|
module SafePgMigrations
|
14
15
|
# Order matters: the bottom-most plugin will have precedence
|
@@ -18,30 +19,25 @@ module SafePgMigrations
|
|
18
19
|
StatementRetrier,
|
19
20
|
StatementInsurer,
|
20
21
|
UselessStatementsLogger,
|
21
|
-
|
22
|
+
Polyfills::IndexDefinitionPolyfill,
|
22
23
|
].freeze
|
23
24
|
|
24
25
|
class << self
|
25
|
-
attr_reader :current_migration
|
26
|
+
attr_reader :current_migration
|
26
27
|
|
27
|
-
def setup_and_teardown(migration, connection)
|
28
|
-
@pg_version_num = get_pg_version_num(connection)
|
28
|
+
def setup_and_teardown(migration, connection, &block)
|
29
29
|
@alternate_connection = nil
|
30
30
|
@current_migration = migration
|
31
31
|
stdout_sql_logger = VerboseSqlLogger.new.setup if verbose?
|
32
32
|
PLUGINS.each { |plugin| connection.extend(plugin) }
|
33
33
|
|
34
|
-
connection.with_setting(:lock_timeout, SafePgMigrations.config.pg_safe_timeout)
|
34
|
+
connection.with_setting(:lock_timeout, SafePgMigrations.config.pg_safe_timeout, &block)
|
35
35
|
ensure
|
36
36
|
close_alternate_connection
|
37
37
|
@current_migration = nil
|
38
38
|
stdout_sql_logger&.teardown
|
39
39
|
end
|
40
40
|
|
41
|
-
def get_pg_version_num(connection)
|
42
|
-
connection.query_value('SHOW server_version_num').to_i
|
43
|
-
end
|
44
|
-
|
45
41
|
def alternate_connection
|
46
42
|
@alternate_connection ||= ActiveRecord::Base.connection_pool.send(:new_connection)
|
47
43
|
end
|
@@ -64,6 +60,9 @@ module SafePgMigrations
|
|
64
60
|
end
|
65
61
|
|
66
62
|
def verbose?
|
63
|
+
unless current_migration.class._safe_pg_migrations_verbose.nil?
|
64
|
+
return current_migration.class._safe_pg_migrations_verbose
|
65
|
+
end
|
67
66
|
return ENV['SAFE_PG_MIGRATIONS_VERBOSE'] == '1' if ENV['SAFE_PG_MIGRATIONS_VERBOSE']
|
68
67
|
return Rails.env.production? if defined?(Rails)
|
69
68
|
|
@@ -76,6 +75,14 @@ module SafePgMigrations
|
|
76
75
|
end
|
77
76
|
|
78
77
|
module Migration
|
78
|
+
module ClassMethods
|
79
|
+
attr_accessor :_safe_pg_migrations_verbose
|
80
|
+
|
81
|
+
def safe_pg_migrations_verbose(verbose)
|
82
|
+
@_safe_pg_migrations_verbose = verbose
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
79
86
|
def exec_migration(connection, direction)
|
80
87
|
SafePgMigrations.setup_and_teardown(self, connection) do
|
81
88
|
super(connection, direction)
|
@@ -4,12 +4,8 @@ require 'active_support/core_ext/numeric/time'
|
|
4
4
|
|
5
5
|
module SafePgMigrations
|
6
6
|
class Configuration
|
7
|
-
attr_accessor :safe_timeout
|
8
|
-
|
9
|
-
attr_accessor :blocking_activity_logger_verbose
|
10
|
-
attr_accessor :batch_size
|
11
|
-
attr_accessor :retry_delay
|
12
|
-
attr_accessor :max_tries
|
7
|
+
attr_accessor :safe_timeout, :blocking_activity_logger_margin, :blocking_activity_logger_verbose, :batch_size,
|
8
|
+
:retry_delay, :max_tries
|
13
9
|
|
14
10
|
def initialize
|
15
11
|
self.safe_timeout = 5.seconds
|
@@ -0,0 +1,59 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SafePgMigrations
|
4
|
+
module Helpers
|
5
|
+
module BlockingActivityFormatter
|
6
|
+
def log_queries(queries)
|
7
|
+
if queries.empty?
|
8
|
+
SafePgMigrations.say 'Could not find any blocking query.', true
|
9
|
+
else
|
10
|
+
SafePgMigrations.say(
|
11
|
+
"Statement was being blocked by the following #{'query'.pluralize(queries.size)}:",
|
12
|
+
true
|
13
|
+
)
|
14
|
+
SafePgMigrations.say '', true
|
15
|
+
output_blocking_queries(queries)
|
16
|
+
SafePgMigrations.say(
|
17
|
+
'Beware, some of those queries might run in a transaction. In this case the locking query might be ' \
|
18
|
+
'located elsewhere in the transaction',
|
19
|
+
true
|
20
|
+
)
|
21
|
+
SafePgMigrations.say '', true
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
def output_blocking_queries(queries)
|
28
|
+
if SafePgMigrations.config.blocking_activity_logger_verbose
|
29
|
+
queries.each do |pid, query, start_time|
|
30
|
+
SafePgMigrations.say "Query with pid #{pid || 'null'} started #{format_start_time start_time}: #{query}",
|
31
|
+
true
|
32
|
+
end
|
33
|
+
else
|
34
|
+
output_confidentially_blocking_queries(queries)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def output_confidentially_blocking_queries(queries)
|
39
|
+
queries.each do |start_time, locktype, mode, pid, transactionid|
|
40
|
+
SafePgMigrations.say(
|
41
|
+
"Query with pid #{pid || 'null'} " \
|
42
|
+
"started #{format_start_time(start_time)}: " \
|
43
|
+
"lock type: #{locktype || 'null'}, " \
|
44
|
+
"lock mode: #{mode || 'null'}, " \
|
45
|
+
"lock transactionid: #{transactionid || 'null'}",
|
46
|
+
true
|
47
|
+
)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def format_start_time(start_time, reference_time = Time.now)
|
52
|
+
start_time = Time.parse(start_time) unless start_time.is_a? Time
|
53
|
+
|
54
|
+
duration = (reference_time - start_time).round
|
55
|
+
"#{duration} #{'second'.pluralize(duration)} ago"
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SafePgMigrations
|
4
|
+
module Helpers
|
5
|
+
module BlockingActivitySelector
|
6
|
+
FILTERED_COLUMNS = %w[
|
7
|
+
blocked_activity.xact_start
|
8
|
+
blocked_locks.locktype
|
9
|
+
blocked_locks.mode
|
10
|
+
blocking_activity.pid
|
11
|
+
blocked_locks.transactionid
|
12
|
+
].freeze
|
13
|
+
|
14
|
+
VERBOSE_COLUMNS = %w[
|
15
|
+
blocking_activity.pid
|
16
|
+
blocking_activity.query
|
17
|
+
blocked_activity.xact_start
|
18
|
+
].freeze
|
19
|
+
|
20
|
+
def select_blocking_queries_sql
|
21
|
+
columns =
|
22
|
+
(
|
23
|
+
if SafePgMigrations.config.blocking_activity_logger_verbose
|
24
|
+
VERBOSE_COLUMNS
|
25
|
+
else
|
26
|
+
FILTERED_COLUMNS
|
27
|
+
end
|
28
|
+
)
|
29
|
+
|
30
|
+
<<~SQL.squish
|
31
|
+
SELECT #{columns.join(', ')}
|
32
|
+
FROM pg_catalog.pg_locks blocked_locks
|
33
|
+
JOIN pg_catalog.pg_stat_activity blocked_activity
|
34
|
+
ON blocked_activity.pid = blocked_locks.pid
|
35
|
+
JOIN pg_catalog.pg_locks blocking_locks
|
36
|
+
ON blocking_locks.locktype = blocked_locks.locktype
|
37
|
+
AND blocking_locks.DATABASE IS NOT DISTINCT FROM blocked_locks.DATABASE
|
38
|
+
AND blocking_locks.relation IS NOT DISTINCT FROM blocked_locks.relation
|
39
|
+
AND blocking_locks.page IS NOT DISTINCT FROM blocked_locks.page
|
40
|
+
AND blocking_locks.tuple IS NOT DISTINCT FROM blocked_locks.tuple
|
41
|
+
AND blocking_locks.virtualxid IS NOT DISTINCT FROM blocked_locks.virtualxid
|
42
|
+
AND blocking_locks.transactionid IS NOT DISTINCT FROM blocked_locks.transactionid
|
43
|
+
AND blocking_locks.classid IS NOT DISTINCT FROM blocked_locks.classid
|
44
|
+
AND blocking_locks.objid IS NOT DISTINCT FROM blocked_locks.objid
|
45
|
+
AND blocking_locks.objsubid IS NOT DISTINCT FROM blocked_locks.objsubid
|
46
|
+
AND blocking_locks.pid != blocked_locks.pid
|
47
|
+
JOIN pg_catalog.pg_stat_activity blocking_activity
|
48
|
+
ON blocking_activity.pid = blocking_locks.pid
|
49
|
+
WHERE blocked_locks.pid = %d
|
50
|
+
SQL
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -1,65 +1,62 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require_relative '../helpers/blocking_activity_formatter'
|
4
|
+
require_relative '../helpers/blocking_activity_selector'
|
5
|
+
|
3
6
|
module SafePgMigrations
|
4
|
-
module BlockingActivityLogger
|
5
|
-
|
6
|
-
|
7
|
-
blocked_locks.locktype
|
8
|
-
blocked_locks.mode
|
9
|
-
blocking_activity.pid
|
10
|
-
blocked_locks.transactionid
|
11
|
-
].freeze
|
12
|
-
|
13
|
-
VERBOSE_COLUMNS = %w[
|
14
|
-
blocking_activity.query
|
15
|
-
blocked_activity.xact_start
|
16
|
-
].freeze
|
7
|
+
module BlockingActivityLogger
|
8
|
+
include ::SafePgMigrations::Helpers::BlockingActivityFormatter
|
9
|
+
include ::SafePgMigrations::Helpers::BlockingActivitySelector
|
17
10
|
|
18
11
|
%i[
|
19
|
-
add_column
|
12
|
+
add_column
|
13
|
+
remove_column
|
14
|
+
add_foreign_key
|
15
|
+
remove_foreign_key
|
16
|
+
change_column_default
|
17
|
+
change_column_null
|
18
|
+
create_table
|
20
19
|
].each do |method|
|
21
20
|
define_method method do |*args, &block|
|
22
|
-
|
21
|
+
log_blocking_queries_after_lock { super(*args, &block) }
|
23
22
|
end
|
24
23
|
ruby2_keywords method
|
25
24
|
end
|
26
25
|
|
27
|
-
|
26
|
+
%i[add_index remove_index].each do |method|
|
27
|
+
define_method method do |*args, **options, &block|
|
28
|
+
return super(*args, **options, &block) if options[:algorithm] != :concurrently
|
28
29
|
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
<<~SQL.squish
|
33
|
-
SELECT #{columns.join(', ')}
|
34
|
-
FROM pg_catalog.pg_locks blocked_locks
|
35
|
-
JOIN pg_catalog.pg_stat_activity blocked_activity
|
36
|
-
ON blocked_activity.pid = blocked_locks.pid
|
37
|
-
JOIN pg_catalog.pg_locks blocking_locks
|
38
|
-
ON blocking_locks.locktype = blocked_locks.locktype
|
39
|
-
AND blocking_locks.DATABASE IS NOT DISTINCT FROM blocked_locks.DATABASE
|
40
|
-
AND blocking_locks.relation IS NOT DISTINCT FROM blocked_locks.relation
|
41
|
-
AND blocking_locks.page IS NOT DISTINCT FROM blocked_locks.page
|
42
|
-
AND blocking_locks.tuple IS NOT DISTINCT FROM blocked_locks.tuple
|
43
|
-
AND blocking_locks.virtualxid IS NOT DISTINCT FROM blocked_locks.virtualxid
|
44
|
-
AND blocking_locks.transactionid IS NOT DISTINCT FROM blocked_locks.transactionid
|
45
|
-
AND blocking_locks.classid IS NOT DISTINCT FROM blocked_locks.classid
|
46
|
-
AND blocking_locks.objid IS NOT DISTINCT FROM blocked_locks.objid
|
47
|
-
AND blocking_locks.objsubid IS NOT DISTINCT FROM blocked_locks.objsubid
|
48
|
-
AND blocking_locks.pid != blocked_locks.pid
|
49
|
-
JOIN pg_catalog.pg_stat_activity blocking_activity
|
50
|
-
ON blocking_activity.pid = blocking_locks.pid
|
51
|
-
WHERE blocked_locks.pid = %d
|
52
|
-
SQL
|
30
|
+
log_blocking_queries_loop { super(*args, **options, &block) }
|
31
|
+
end
|
53
32
|
end
|
54
33
|
|
55
|
-
|
56
|
-
|
57
|
-
|
34
|
+
private
|
35
|
+
|
36
|
+
def log_blocking_queries_loop
|
37
|
+
blocking_queries_retriever_thread =
|
38
|
+
Thread.new do
|
39
|
+
loop do
|
40
|
+
sleep SafePgMigrations.config.retry_delay
|
41
|
+
|
42
|
+
log_queries SafePgMigrations.alternate_connection.query(
|
43
|
+
select_blocking_queries_sql % raw_connection.backend_pid
|
44
|
+
)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
yield
|
49
|
+
|
50
|
+
blocking_queries_retriever_thread.kill
|
51
|
+
end
|
58
52
|
|
53
|
+
def log_blocking_queries_after_lock
|
59
54
|
blocking_queries_retriever_thread =
|
60
55
|
Thread.new do
|
61
56
|
sleep delay_before_logging
|
62
|
-
SafePgMigrations.alternate_connection.query(
|
57
|
+
SafePgMigrations.alternate_connection.query(
|
58
|
+
select_blocking_queries_sql % raw_connection.backend_pid
|
59
|
+
)
|
63
60
|
end
|
64
61
|
|
65
62
|
yield
|
@@ -71,51 +68,25 @@ module SafePgMigrations
|
|
71
68
|
begin
|
72
69
|
blocking_queries_retriever_thread.value
|
73
70
|
rescue StandardError => e
|
74
|
-
SafePgMigrations.say(
|
71
|
+
SafePgMigrations.say(
|
72
|
+
"Error while retrieving blocking queries: #{e}",
|
73
|
+
true
|
74
|
+
)
|
75
75
|
nil
|
76
76
|
end
|
77
77
|
|
78
|
-
|
79
|
-
|
80
|
-
if queries.empty?
|
81
|
-
SafePgMigrations.say 'Could not find any blocking query.', true
|
82
|
-
else
|
83
|
-
SafePgMigrations.say(
|
84
|
-
"Statement was being blocked by the following #{'query'.pluralize(queries.size)}:", true
|
85
|
-
)
|
86
|
-
SafePgMigrations.say '', true
|
87
|
-
output_blocking_queries(queries)
|
88
|
-
SafePgMigrations.say(
|
89
|
-
'Beware, some of those queries might run in a transaction. In this case the locking query might be '\
|
90
|
-
'located elsewhere in the transaction',
|
91
|
-
true
|
92
|
-
)
|
93
|
-
SafePgMigrations.say '', true
|
94
|
-
end
|
78
|
+
log_queries queries unless queries.nil?
|
95
79
|
|
96
80
|
raise
|
97
81
|
end
|
98
82
|
|
99
|
-
def
|
100
|
-
|
101
|
-
|
102
|
-
else
|
103
|
-
queries.each do |start_time, locktype, mode, pid, transactionid|
|
104
|
-
SafePgMigrations.say(
|
105
|
-
"#{format_start_time(start_time)}: lock type: #{locktype || 'null'}, " \
|
106
|
-
"lock mode: #{mode || 'null'}, " \
|
107
|
-
"lock pid: #{pid || 'null'}, " \
|
108
|
-
"lock transactionid: #{transactionid || 'null'}",
|
109
|
-
true
|
110
|
-
)
|
111
|
-
end
|
112
|
-
end
|
83
|
+
def delay_before_logging
|
84
|
+
SafePgMigrations.config.safe_timeout -
|
85
|
+
SafePgMigrations.config.blocking_activity_logger_margin
|
113
86
|
end
|
114
87
|
|
115
|
-
def
|
116
|
-
|
117
|
-
duration = (reference_time - start_time).round
|
118
|
-
"transaction started #{duration} #{'second'.pluralize(duration)} ago"
|
88
|
+
def delay_before_retry
|
89
|
+
SafePgMigrations.config.blocking_activity_logger_margin + SafePgMigrations.config.retry_delay
|
119
90
|
end
|
120
91
|
end
|
121
92
|
end
|
@@ -53,6 +53,16 @@ module SafePgMigrations
|
|
53
53
|
)
|
54
54
|
end
|
55
55
|
|
56
|
+
def remove_foreign_key(from_table, to_table = nil, **options)
|
57
|
+
return super if foreign_key_exists?(from_table, to_table, **options)
|
58
|
+
|
59
|
+
reference_name = to_table || options[:to_table] || options[:column] || options[:name]
|
60
|
+
SafePgMigrations.say(
|
61
|
+
"/!\\ Foreign key '#{from_table}' -> '#{reference_name}' does not exist. Skipping statement.",
|
62
|
+
true
|
63
|
+
)
|
64
|
+
end
|
65
|
+
|
56
66
|
ruby2_keywords def create_table(table_name, *args)
|
57
67
|
options = args.last.is_a?(Hash) ? args.last : {}
|
58
68
|
return super if options[:force] || !table_exists?(table_name)
|
@@ -2,8 +2,6 @@
|
|
2
2
|
|
3
3
|
module SafePgMigrations
|
4
4
|
module StatementInsurer
|
5
|
-
PG_11_VERSION_NUM = 110_000
|
6
|
-
|
7
5
|
%i[change_column_null change_column].each do |method|
|
8
6
|
define_method method do |*args, &block|
|
9
7
|
with_setting(:statement_timeout, SafePgMigrations.config.pg_safe_timeout) { super(*args, &block) }
|
@@ -11,33 +9,6 @@ module SafePgMigrations
|
|
11
9
|
ruby2_keywords method
|
12
10
|
end
|
13
11
|
|
14
|
-
ruby2_keywords def add_column(table_name, column_name, type, *args) # rubocop:disable Metrics/CyclomaticComplexity
|
15
|
-
options = args.last.is_a?(Hash) ? args.last : {}
|
16
|
-
return super if SafePgMigrations.pg_version_num >= PG_11_VERSION_NUM
|
17
|
-
|
18
|
-
default = options.delete(:default)
|
19
|
-
null = options.delete(:null)
|
20
|
-
|
21
|
-
if !default.nil? || null == false
|
22
|
-
SafePgMigrations.say_method_call(:add_column, table_name, column_name, type, options)
|
23
|
-
end
|
24
|
-
|
25
|
-
super
|
26
|
-
|
27
|
-
unless default.nil?
|
28
|
-
SafePgMigrations.say_method_call(:change_column_default, table_name, column_name, default)
|
29
|
-
change_column_default(table_name, column_name, default)
|
30
|
-
|
31
|
-
SafePgMigrations.say_method_call(:backfill_column_default, table_name, column_name)
|
32
|
-
backfill_column_default(table_name, column_name)
|
33
|
-
end
|
34
|
-
|
35
|
-
if null == false # rubocop:disable Style/GuardClause
|
36
|
-
SafePgMigrations.say_method_call(:change_column_null, table_name, column_name, null)
|
37
|
-
change_column_null(table_name, column_name, null)
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
12
|
ruby2_keywords def add_foreign_key(from_table, to_table, *args)
|
42
13
|
options = args.last.is_a?(Hash) ? args.last : {}
|
43
14
|
validate_present = options.key? :validate
|
@@ -64,7 +35,9 @@ module SafePgMigrations
|
|
64
35
|
end
|
65
36
|
end
|
66
37
|
|
67
|
-
def add_index(table_name, column_name,
|
38
|
+
ruby2_keywords def add_index(table_name, column_name, *args_options)
|
39
|
+
options = args_options.last.is_a?(Hash) ? args_options.last : {}
|
40
|
+
|
68
41
|
if options[:algorithm] == :default
|
69
42
|
options.delete :algorithm
|
70
43
|
else
|
@@ -84,15 +57,6 @@ module SafePgMigrations
|
|
84
57
|
without_timeout { super(table_name, **options) }
|
85
58
|
end
|
86
59
|
|
87
|
-
def backfill_column_default(table_name, column_name)
|
88
|
-
model = Class.new(ActiveRecord::Base) { self.table_name = table_name }
|
89
|
-
quoted_column_name = quote_column_name(column_name)
|
90
|
-
|
91
|
-
model.in_batches(of: SafePgMigrations.config.batch_size).each do |relation|
|
92
|
-
relation.update_all("#{quoted_column_name} = DEFAULT")
|
93
|
-
end
|
94
|
-
end
|
95
|
-
|
96
60
|
def with_setting(key, value)
|
97
61
|
old_value = query_value("SHOW #{key}")
|
98
62
|
execute("SET #{key} TO #{quote(value)}")
|
@@ -109,16 +73,16 @@ module SafePgMigrations
|
|
109
73
|
end
|
110
74
|
end
|
111
75
|
|
112
|
-
def without_statement_timeout
|
113
|
-
with_setting(:statement_timeout, 0)
|
76
|
+
def without_statement_timeout(&block)
|
77
|
+
with_setting(:statement_timeout, 0, &block)
|
114
78
|
end
|
115
79
|
|
116
|
-
def without_lock_timeout
|
117
|
-
with_setting(:lock_timeout, 0)
|
80
|
+
def without_lock_timeout(&block)
|
81
|
+
with_setting(:lock_timeout, 0, &block)
|
118
82
|
end
|
119
83
|
|
120
|
-
def without_timeout
|
121
|
-
without_statement_timeout { without_lock_timeout
|
84
|
+
def without_timeout(&block)
|
85
|
+
without_statement_timeout { without_lock_timeout(&block) }
|
122
86
|
end
|
123
87
|
end
|
124
88
|
end
|
@@ -4,7 +4,7 @@ module SafePgMigrations
|
|
4
4
|
class VerboseSqlLogger
|
5
5
|
def setup
|
6
6
|
@activerecord_logger_was = ActiveRecord::Base.logger
|
7
|
-
@verbose_query_logs_was =
|
7
|
+
@verbose_query_logs_was = Polyfills::VerboseQueryLogsPolyfill.verbose_query_logs
|
8
8
|
@colorize_logging_was = ActiveRecord::LogSubscriber.colorize_logging
|
9
9
|
|
10
10
|
disable_marginalia if defined?(Marginalia)
|
@@ -13,12 +13,12 @@ module SafePgMigrations
|
|
13
13
|
ActiveRecord::Base.logger = stdout_logger
|
14
14
|
ActiveRecord::LogSubscriber.colorize_logging = colorize_logging?
|
15
15
|
# Do not output caller method, we know it is coming from the migration
|
16
|
-
|
16
|
+
Polyfills::VerboseQueryLogsPolyfill.verbose_query_logs = false
|
17
17
|
self
|
18
18
|
end
|
19
19
|
|
20
20
|
def teardown
|
21
|
-
|
21
|
+
Polyfills::VerboseQueryLogsPolyfill.verbose_query_logs = @verbose_query_logs_was
|
22
22
|
ActiveRecord::LogSubscriber.colorize_logging = @colorize_logging_was
|
23
23
|
ActiveRecord::Base.logger = @activerecord_logger_was
|
24
24
|
enable_marginalia if defined?(Marginalia)
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SafePgMigrations
|
4
|
+
module Polyfills
|
5
|
+
module IndexDefinitionPolyfill
|
6
|
+
include SatisfiedHelper
|
7
|
+
|
8
|
+
protected
|
9
|
+
|
10
|
+
IndexDefinition = Struct.new(:table, :name)
|
11
|
+
|
12
|
+
def index_definition(table_name, column_name, **options)
|
13
|
+
return super(table_name, column_name, **options) if satisfied? '>=6.1.0'
|
14
|
+
|
15
|
+
index_name = options.key?(:name) ? options[:name].to_s : index_name(table_name, index_column_names(column_name))
|
16
|
+
validate_index_length!(table_name, index_name, options.fetch(:internal, false))
|
17
|
+
|
18
|
+
IndexDefinition.new(table_name, index_name)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SafePgMigrations
|
4
|
+
module Polyfills
|
5
|
+
module SatisfiedHelper
|
6
|
+
private
|
7
|
+
|
8
|
+
def satisfied?(version)
|
9
|
+
Gem::Requirement.new(version).satisfied_by? Gem::Version.new(::ActiveRecord::VERSION::STRING)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SafePgMigrations
|
4
|
+
module Polyfills
|
5
|
+
module VerboseQueryLogsPolyfill
|
6
|
+
class << self
|
7
|
+
include SatisfiedHelper
|
8
|
+
|
9
|
+
def verbose_query_logs
|
10
|
+
return ActiveRecord.verbose_query_logs if satisfied? '>=7.0.0'
|
11
|
+
|
12
|
+
ActiveRecord::Base.verbose_query_logs
|
13
|
+
end
|
14
|
+
|
15
|
+
def verbose_query_logs=(value)
|
16
|
+
if satisfied? '>=7.0.0'
|
17
|
+
ActiveRecord.verbose_query_logs = value
|
18
|
+
return
|
19
|
+
end
|
20
|
+
|
21
|
+
ActiveRecord::Base.verbose_query_logs = value
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -7,6 +7,7 @@ module SafePgMigrations
|
|
7
7
|
initializer 'safe_pg_migrations.insert_into_active_record' do
|
8
8
|
ActiveSupport.on_load :active_record do
|
9
9
|
ActiveRecord::Migration.prepend(SafePgMigrations::Migration)
|
10
|
+
ActiveRecord::Migration.singleton_class.prepend(SafePgMigrations::Migration::ClassMethods)
|
10
11
|
end
|
11
12
|
end
|
12
13
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: safe-pg-migrations
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Matthieu Prat
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date:
|
13
|
+
date: 2023-03-17 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: activerecord
|
@@ -18,42 +18,28 @@ dependencies:
|
|
18
18
|
requirements:
|
19
19
|
- - ">="
|
20
20
|
- !ruby/object:Gem::Version
|
21
|
-
version: '
|
21
|
+
version: '6.0'
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
24
|
version_requirements: !ruby/object:Gem::Requirement
|
25
25
|
requirements:
|
26
26
|
- - ">="
|
27
27
|
- !ruby/object:Gem::Version
|
28
|
-
version: '
|
28
|
+
version: '6.0'
|
29
29
|
- !ruby/object:Gem::Dependency
|
30
30
|
name: activesupport
|
31
31
|
requirement: !ruby/object:Gem::Requirement
|
32
32
|
requirements:
|
33
33
|
- - ">="
|
34
34
|
- !ruby/object:Gem::Version
|
35
|
-
version: '
|
35
|
+
version: '6.0'
|
36
36
|
type: :runtime
|
37
37
|
prerelease: false
|
38
38
|
version_requirements: !ruby/object:Gem::Requirement
|
39
39
|
requirements:
|
40
40
|
- - ">="
|
41
41
|
- !ruby/object:Gem::Version
|
42
|
-
version: '
|
43
|
-
- !ruby/object:Gem::Dependency
|
44
|
-
name: ruby2_keywords
|
45
|
-
requirement: !ruby/object:Gem::Requirement
|
46
|
-
requirements:
|
47
|
-
- - ">="
|
48
|
-
- !ruby/object:Gem::Version
|
49
|
-
version: 0.0.4
|
50
|
-
type: :runtime
|
51
|
-
prerelease: false
|
52
|
-
version_requirements: !ruby/object:Gem::Requirement
|
53
|
-
requirements:
|
54
|
-
- - ">="
|
55
|
-
- !ruby/object:Gem::Version
|
56
|
-
version: 0.0.4
|
42
|
+
version: '6.0'
|
57
43
|
description: Make your PG migrations safe.
|
58
44
|
email:
|
59
45
|
executables: []
|
@@ -65,13 +51,17 @@ files:
|
|
65
51
|
- lib/safe-pg-migrations.rb
|
66
52
|
- lib/safe-pg-migrations/base.rb
|
67
53
|
- lib/safe-pg-migrations/configuration.rb
|
54
|
+
- lib/safe-pg-migrations/helpers/blocking_activity_formatter.rb
|
55
|
+
- lib/safe-pg-migrations/helpers/blocking_activity_selector.rb
|
68
56
|
- lib/safe-pg-migrations/plugins/blocking_activity_logger.rb
|
69
57
|
- lib/safe-pg-migrations/plugins/idempotent_statements.rb
|
70
|
-
- lib/safe-pg-migrations/plugins/legacy_active_record_support.rb
|
71
58
|
- lib/safe-pg-migrations/plugins/statement_insurer.rb
|
72
59
|
- lib/safe-pg-migrations/plugins/statement_retrier.rb
|
73
60
|
- lib/safe-pg-migrations/plugins/useless_statements_logger.rb
|
74
61
|
- lib/safe-pg-migrations/plugins/verbose_sql_logger.rb
|
62
|
+
- lib/safe-pg-migrations/polyfills/index_definition_polyfill.rb
|
63
|
+
- lib/safe-pg-migrations/polyfills/satisfied_helper.rb
|
64
|
+
- lib/safe-pg-migrations/polyfills/verbose_query_logs_polyfill.rb
|
75
65
|
- lib/safe-pg-migrations/railtie.rb
|
76
66
|
- lib/safe-pg-migrations/version.rb
|
77
67
|
homepage: https://github.com/doctolib/safe-pg-migrations
|
@@ -83,6 +73,7 @@ metadata:
|
|
83
73
|
mailing_list_uri: https://doctolib.engineering/engineering-news-ruby-rails-react
|
84
74
|
source_code_uri: https://github.com/doctolib/safe-pg-migrations
|
85
75
|
contributors_uri: https://github.com/doctolib/safe-pg-migrations/graphs/contributors
|
76
|
+
rubygems_mfa_required: 'true'
|
86
77
|
post_install_message:
|
87
78
|
rdoc_options: []
|
88
79
|
require_paths:
|
@@ -91,18 +82,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
91
82
|
requirements:
|
92
83
|
- - ">="
|
93
84
|
- !ruby/object:Gem::Version
|
94
|
-
version: '2.
|
95
|
-
- - "<"
|
96
|
-
- !ruby/object:Gem::Version
|
97
|
-
version: '4'
|
85
|
+
version: '2.7'
|
98
86
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
99
87
|
requirements:
|
100
88
|
- - ">="
|
101
89
|
- !ruby/object:Gem::Version
|
102
90
|
version: '0'
|
103
91
|
requirements: []
|
104
|
-
|
105
|
-
rubygems_version: 2.7.3
|
92
|
+
rubygems_version: 3.3.7
|
106
93
|
signing_key:
|
107
94
|
specification_version: 4
|
108
95
|
summary: Make your PG migrations safe.
|
@@ -1,38 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module SafePgMigrations
|
4
|
-
module LegacyActiveRecordSupport
|
5
|
-
ruby2_keywords def validate_foreign_key(*args)
|
6
|
-
return super(*args) if satisfied? '>=6.0.0'
|
7
|
-
|
8
|
-
from_table, to_table, options = args
|
9
|
-
super(from_table, to_table || options)
|
10
|
-
end
|
11
|
-
|
12
|
-
ruby2_keywords def foreign_key_exists?(*args)
|
13
|
-
return super(*args) if satisfied? '>=6.0.0'
|
14
|
-
|
15
|
-
from_table, to_table, options = args
|
16
|
-
super(from_table, to_table || options)
|
17
|
-
end
|
18
|
-
|
19
|
-
protected
|
20
|
-
|
21
|
-
IndexDefinition = Struct.new(:table, :name)
|
22
|
-
|
23
|
-
def index_definition(table_name, column_name, **options)
|
24
|
-
return super(table_name, column_name, **options) if satisfied? '>=6.1.0'
|
25
|
-
|
26
|
-
index_name = options.key?(:name) ? options[:name].to_s : index_name(table_name, index_column_names(column_name))
|
27
|
-
validate_index_length!(table_name, index_name, options.fetch(:internal, false))
|
28
|
-
|
29
|
-
IndexDefinition.new(table_name, index_name)
|
30
|
-
end
|
31
|
-
|
32
|
-
private
|
33
|
-
|
34
|
-
def satisfied?(version)
|
35
|
-
Gem::Requirement.new(version).satisfied_by? Gem::Version.new(::ActiveRecord::VERSION::STRING)
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|