online_migrations 0.2.0 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.rubocop.yml +8 -1
- data/CHANGELOG.md +15 -0
- data/Gemfile.lock +22 -22
- data/README.md +105 -2
- data/lib/generators/online_migrations/templates/initializer.rb.tt +9 -0
- data/lib/online_migrations/background_migrations/background_migration_class_validator.rb +1 -1
- data/lib/online_migrations/background_migrations/copy_column.rb +1 -1
- data/lib/online_migrations/change_column_type_helpers.rb +2 -2
- data/lib/online_migrations/command_checker.rb +34 -7
- data/lib/online_migrations/config.rb +14 -0
- data/lib/online_migrations/error_messages.rb +25 -0
- data/lib/online_migrations/lock_retrier.rb +1 -1
- data/lib/online_migrations/migration.rb +24 -8
- data/lib/online_migrations/schema_statements.rb +27 -19
- data/lib/online_migrations/verbose_sql_logs.rb +45 -0
- data/lib/online_migrations/version.rb +1 -1
- data/lib/online_migrations.rb +1 -0
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4706d65d4ed40219b3680266294f9382d1db51ac168768b77f55e1a73a2d3de3
|
4
|
+
data.tar.gz: b828d4c3d8cc9ef1268eeaa00f86bc3fda41546963334cb356916b1ad481b3bc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f70ab747e025f34a546da3d97aad6d7a175006fca9c48a3ae402b1683715b1eeb5128213bec52612c9178e457a638f9eb7d370445cf43b7c164f6d9bb548026f
|
7
|
+
data.tar.gz: e001963c1fc82ec925a4b1c917689171e35dc3c4f0fba61f8a29ec0e2f5b2fedefa6733828d81176b489a345113e054f5f174c007152fc407d55e3184853cc77
|
data/.gitignore
CHANGED
data/.rubocop.yml
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
AllCops:
|
2
|
-
NewCops:
|
2
|
+
NewCops: enable
|
3
3
|
SuggestExtensions: false
|
4
4
|
|
5
5
|
Style/StringLiterals:
|
@@ -68,6 +68,10 @@ Style/WhileUntilModifier:
|
|
68
68
|
Style/HashAsLastArrayItem:
|
69
69
|
Enabled: false
|
70
70
|
|
71
|
+
# only for ruby 2.6+
|
72
|
+
Style/MapToHash:
|
73
|
+
Enabled: false
|
74
|
+
|
71
75
|
# we can not use new syntax for older rubies
|
72
76
|
Lint/ErbNewArguments:
|
73
77
|
Enabled: false
|
@@ -109,5 +113,8 @@ Naming/AccessorMethodName:
|
|
109
113
|
Gemspec/RequiredRubyVersion:
|
110
114
|
Enabled: false
|
111
115
|
|
116
|
+
Gemspec/RequireMFA:
|
117
|
+
Enabled: false
|
118
|
+
|
112
119
|
Metrics:
|
113
120
|
Enabled: false
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,20 @@
|
|
1
1
|
## master (unreleased)
|
2
2
|
|
3
|
+
## 0.3.0 (2022-02-10)
|
4
|
+
|
5
|
+
- Support ActiveRecord 7.0+ versioned schemas
|
6
|
+
|
7
|
+
- Check for addition of single table inheritance column
|
8
|
+
|
9
|
+
See [Adding a single table inheritance column](https://github.com/fatkodima/online_migrations#adding-a-single-table-inheritance-column) for details
|
10
|
+
|
11
|
+
- Add a way to log every SQL query to stdout
|
12
|
+
|
13
|
+
See [Verbose SQL logs](https://github.com/fatkodima/online_migrations#verbose-sql-logs) for details
|
14
|
+
|
15
|
+
- Ignore new tables when checking for removing table with multiple fkeys
|
16
|
+
- Fix backfilling column in add_column_with_default when default is an expression
|
17
|
+
|
3
18
|
## 0.2.0 (2022-01-31)
|
4
19
|
|
5
20
|
- Check removing a table with multiple foreign keys
|
data/Gemfile.lock
CHANGED
@@ -1,31 +1,31 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
online_migrations (0.
|
4
|
+
online_migrations (0.3.0)
|
5
5
|
activerecord (>= 4.2)
|
6
6
|
|
7
7
|
GEM
|
8
8
|
remote: https://rubygems.org/
|
9
9
|
specs:
|
10
|
-
actionpack (7.0.
|
11
|
-
actionview (= 7.0.
|
12
|
-
activesupport (= 7.0.
|
10
|
+
actionpack (7.0.2)
|
11
|
+
actionview (= 7.0.2)
|
12
|
+
activesupport (= 7.0.2)
|
13
13
|
rack (~> 2.0, >= 2.2.0)
|
14
14
|
rack-test (>= 0.6.3)
|
15
15
|
rails-dom-testing (~> 2.0)
|
16
16
|
rails-html-sanitizer (~> 1.0, >= 1.2.0)
|
17
|
-
actionview (7.0.
|
18
|
-
activesupport (= 7.0.
|
17
|
+
actionview (7.0.2)
|
18
|
+
activesupport (= 7.0.2)
|
19
19
|
builder (~> 3.1)
|
20
20
|
erubi (~> 1.4)
|
21
21
|
rails-dom-testing (~> 2.0)
|
22
22
|
rails-html-sanitizer (~> 1.1, >= 1.2.0)
|
23
|
-
activemodel (7.0.
|
24
|
-
activesupport (= 7.0.
|
25
|
-
activerecord (7.0.
|
26
|
-
activemodel (= 7.0.
|
27
|
-
activesupport (= 7.0.
|
28
|
-
activesupport (7.0.
|
23
|
+
activemodel (7.0.2)
|
24
|
+
activesupport (= 7.0.2)
|
25
|
+
activerecord (7.0.2)
|
26
|
+
activemodel (= 7.0.2)
|
27
|
+
activesupport (= 7.0.2)
|
28
|
+
activesupport (7.0.2)
|
29
29
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
30
30
|
i18n (>= 1.6, < 2)
|
31
31
|
minitest (>= 5.1)
|
@@ -35,7 +35,7 @@ GEM
|
|
35
35
|
concurrent-ruby (1.1.9)
|
36
36
|
crass (1.0.6)
|
37
37
|
erubi (1.10.0)
|
38
|
-
i18n (1.
|
38
|
+
i18n (1.9.1)
|
39
39
|
concurrent-ruby (~> 1.0)
|
40
40
|
loofah (2.13.0)
|
41
41
|
crass (~> 1.0.2)
|
@@ -43,13 +43,13 @@ GEM
|
|
43
43
|
method_source (1.0.0)
|
44
44
|
mini_portile2 (2.7.1)
|
45
45
|
minitest (5.15.0)
|
46
|
-
nokogiri (1.13.
|
46
|
+
nokogiri (1.13.1)
|
47
47
|
mini_portile2 (~> 2.7.0)
|
48
48
|
racc (~> 1.4)
|
49
49
|
parallel (1.21.0)
|
50
50
|
parser (3.1.0.0)
|
51
51
|
ast (~> 2.4.1)
|
52
|
-
pg (1.
|
52
|
+
pg (1.3.1)
|
53
53
|
racc (1.6.0)
|
54
54
|
rack (2.2.3)
|
55
55
|
rack-test (1.1.0)
|
@@ -59,20 +59,20 @@ GEM
|
|
59
59
|
nokogiri (>= 1.6)
|
60
60
|
rails-html-sanitizer (1.4.2)
|
61
61
|
loofah (~> 2.3)
|
62
|
-
railties (7.0.
|
63
|
-
actionpack (= 7.0.
|
64
|
-
activesupport (= 7.0.
|
62
|
+
railties (7.0.2)
|
63
|
+
actionpack (= 7.0.2)
|
64
|
+
activesupport (= 7.0.2)
|
65
65
|
method_source
|
66
66
|
rake (>= 12.2)
|
67
67
|
thor (~> 1.0)
|
68
68
|
zeitwerk (~> 2.5)
|
69
|
-
rainbow (3.
|
69
|
+
rainbow (3.1.1)
|
70
70
|
rake (12.3.3)
|
71
71
|
regexp_parser (2.2.0)
|
72
72
|
rexml (3.2.5)
|
73
|
-
rubocop (1.
|
73
|
+
rubocop (1.25.1)
|
74
74
|
parallel (~> 1.10)
|
75
|
-
parser (>= 3.
|
75
|
+
parser (>= 3.1.0.0)
|
76
76
|
rainbow (>= 2.2.2, < 4.0)
|
77
77
|
regexp_parser (>= 1.8, < 3.0)
|
78
78
|
rexml
|
@@ -89,7 +89,7 @@ GEM
|
|
89
89
|
webrick (1.7.0)
|
90
90
|
yard (0.9.27)
|
91
91
|
webrick (~> 1.7.0)
|
92
|
-
zeitwerk (2.5.
|
92
|
+
zeitwerk (2.5.4)
|
93
93
|
|
94
94
|
PLATFORMS
|
95
95
|
ruby
|
data/README.md
CHANGED
@@ -138,6 +138,7 @@ Potentially dangerous operations:
|
|
138
138
|
- [adding multiple foreign keys](#adding-multiple-foreign-keys)
|
139
139
|
- [removing a table with multiple foreign keys](#removing-a-table-with-multiple-foreign-keys)
|
140
140
|
- [mismatched reference column types](#mismatched-reference-column-types)
|
141
|
+
- [adding a single table inheritance column](#adding-a-single-table-inheritance-column)
|
141
142
|
|
142
143
|
You can also add [custom checks](#custom-checks) or [disable specific checks](#disable-checks).
|
143
144
|
|
@@ -160,12 +161,12 @@ end
|
|
160
161
|
1. Ignore the column:
|
161
162
|
|
162
163
|
```ruby
|
163
|
-
# For
|
164
|
+
# For ActiveRecord 5+
|
164
165
|
class User < ApplicationRecord
|
165
166
|
self.ignored_columns = ["name"]
|
166
167
|
end
|
167
168
|
|
168
|
-
# For
|
169
|
+
# For ActiveRecord < 5
|
169
170
|
class User < ActiveRecord::Base
|
170
171
|
def self.columns
|
171
172
|
super.reject { |c| c.name == "name" }
|
@@ -978,6 +979,46 @@ class AddUserIdToProjects < ActiveRecord::Migration[7.0]
|
|
978
979
|
end
|
979
980
|
```
|
980
981
|
|
982
|
+
### Adding a single table inheritance column
|
983
|
+
|
984
|
+
:x: **Bad**
|
985
|
+
|
986
|
+
Adding a single table inheritance column might cause errors in old instances of your application.
|
987
|
+
|
988
|
+
```ruby
|
989
|
+
class AddTypeToUsers < ActiveRecord::Migration[7.0]
|
990
|
+
def change
|
991
|
+
add_column :users, :string, :type, default: "Member"
|
992
|
+
end
|
993
|
+
end
|
994
|
+
```
|
995
|
+
|
996
|
+
After the migration was ran and the column was added, but before the code is fully deployed to all instances, an old instance may be restarted (due to an error etc). And when it will fetch 'User' records from the database, 'User' will look for a 'Member' subclass (from the 'type' column) and fail to locate it unless it is already defined.
|
997
|
+
|
998
|
+
:white_check_mark: **Good**
|
999
|
+
|
1000
|
+
A safer approach is to:
|
1001
|
+
|
1002
|
+
1. ignore the column:
|
1003
|
+
|
1004
|
+
```ruby
|
1005
|
+
# For ActiveRecord 5+
|
1006
|
+
class User < ApplicationRecord
|
1007
|
+
self.ignored_columns = ["type"]
|
1008
|
+
end
|
1009
|
+
|
1010
|
+
# For ActiveRecord < 5
|
1011
|
+
class User < ActiveRecord::Base
|
1012
|
+
def self.columns
|
1013
|
+
super.reject { |c| c.name == "type" }
|
1014
|
+
end
|
1015
|
+
end
|
1016
|
+
```
|
1017
|
+
|
1018
|
+
2. deploy
|
1019
|
+
3. remove the column ignoring from step 1 and apply initial code changes
|
1020
|
+
4. deploy
|
1021
|
+
|
981
1022
|
## Assuring Safety
|
982
1023
|
|
983
1024
|
To mark a step in the migration as safe, despite using a method that might otherwise be dangerous, wrap it in a `safety_assured` block.
|
@@ -1139,6 +1180,68 @@ To mark tables as small:
|
|
1139
1180
|
config.small_tables = [:settings, :prices]
|
1140
1181
|
```
|
1141
1182
|
|
1183
|
+
### Verbose SQL logs
|
1184
|
+
|
1185
|
+
For any operation, **Online Migrations** can output the performed SQL queries.
|
1186
|
+
|
1187
|
+
This is useful to demystify `online_migrations` inner workings, and to better investigate migration failure in production. This is also useful in development to get a better grasp of what is going on for high-level statements like `add_column_with_default`.
|
1188
|
+
|
1189
|
+
Consider migration, running on PostgreSQL < 11:
|
1190
|
+
|
1191
|
+
```ruby
|
1192
|
+
class AddAdminToUsers < ActiveRecord::Migration[7.0]
|
1193
|
+
disable_ddl_transaction!
|
1194
|
+
|
1195
|
+
def change
|
1196
|
+
add_column_with_default :users, :admin, :boolean, default: false
|
1197
|
+
end
|
1198
|
+
end
|
1199
|
+
```
|
1200
|
+
|
1201
|
+
Instead of the traditional output:
|
1202
|
+
|
1203
|
+
```
|
1204
|
+
== 20220106214827 AddAdminToUsers: migrating ==================================
|
1205
|
+
-- add_column_with_default(:users, :admin, :boolean, {:default=>false})
|
1206
|
+
-> 0.1423s
|
1207
|
+
== 20220106214827 AddAdminToUsers: migrated (0.1462s) =========================
|
1208
|
+
```
|
1209
|
+
|
1210
|
+
**Online Migrations** will output the following logs:
|
1211
|
+
|
1212
|
+
```
|
1213
|
+
== 20220106214827 AddAdminToUsers: migrating ==================================
|
1214
|
+
(0.3ms) SHOW lock_timeout
|
1215
|
+
(0.2ms) SET lock_timeout TO '50ms'
|
1216
|
+
-- add_column_with_default(:users, :admin, :boolean, {:default=>false})
|
1217
|
+
TRANSACTION (0.1ms) BEGIN
|
1218
|
+
(37.7ms) ALTER TABLE "users" ADD "admin" boolean DEFAULT NULL
|
1219
|
+
(0.5ms) ALTER TABLE "users" ALTER COLUMN "admin" SET DEFAULT FALSE
|
1220
|
+
TRANSACTION (0.3ms) COMMIT
|
1221
|
+
Load (0.3ms) SELECT "users"."id" FROM "users" WHERE ("users"."admin" != FALSE OR "users"."admin" IS NULL) ORDER BY "users"."id" ASC LIMIT $1 [["LIMIT", 1]]
|
1222
|
+
Load (0.5ms) SELECT "users"."id" FROM "users" WHERE ("users"."admin" != FALSE OR "users"."admin" IS NULL) AND "users"."id" >= 1 ORDER BY "users"."id" ASC LIMIT $1 OFFSET $2 [["LIMIT", 1], ["OFFSET", 1000]]
|
1223
|
+
#<Class:0x00007f8ae3703f08> Update All (9.6ms) UPDATE "users" SET "admin" = $1 WHERE ("users"."admin" != FALSE OR "users"."admin" IS NULL) AND "users"."id" >= 1 AND "users"."id" < 1001 [["admin", false]]
|
1224
|
+
Load (0.8ms) SELECT "users"."id" FROM "users" WHERE ("users"."admin" != FALSE OR "users"."admin" IS NULL) AND "users"."id" >= 1001 ORDER BY "users"."id" ASC LIMIT $1 OFFSET $2 [["LIMIT", 1], ["OFFSET", 1000]]
|
1225
|
+
#<Class:0x00007f8ae3703f08> Update All (1.5ms) UPDATE "users" SET "admin" = $1 WHERE ("users"."admin" != FALSE OR "users"."admin" IS NULL) AND "users"."id" >= 1001 [["admin", false]]
|
1226
|
+
-> 0.1814s
|
1227
|
+
(0.4ms) SET lock_timeout TO '5s'
|
1228
|
+
== 20220106214827 AddAdminToUsers: migrated (0.1840s) =========================
|
1229
|
+
```
|
1230
|
+
|
1231
|
+
So you can actually check which steps are performed.
|
1232
|
+
|
1233
|
+
**Note**: The `SHOW` statements are used by **Online Migrations** to query settings for their original values in order to restore them after the work is done.
|
1234
|
+
|
1235
|
+
To enable verbose sql logs:
|
1236
|
+
|
1237
|
+
```ruby
|
1238
|
+
# config/initializers/online_migrations.rb
|
1239
|
+
|
1240
|
+
config.verbose_sql_logs = true
|
1241
|
+
```
|
1242
|
+
|
1243
|
+
This feature is enabled by default in a production Rails environment. You can override this setting via `ONLINE_MIGRATIONS_VERBOSE_SQL_LOGS` environment variable.
|
1244
|
+
|
1142
1245
|
## Background Migrations
|
1143
1246
|
|
1144
1247
|
Read [BACKGROUND_MIGRATIONS.md](BACKGROUND_MIGRATIONS.md) on how to perform data migrations on large tables.
|
@@ -32,6 +32,15 @@ OnlineMigrations.configure do |config|
|
|
32
32
|
# For the list of available checks look at `lib/error_messages` folder.
|
33
33
|
# config.enable_check(:remove_index)
|
34
34
|
|
35
|
+
# Configure whether to log every SQL query happening in a migration.
|
36
|
+
#
|
37
|
+
# This is useful to demystify online_migrations inner workings, and to better investigate
|
38
|
+
# migration failure in production. This is also useful in development to get
|
39
|
+
# a better grasp of what is going on for high-level statements like add_column_with_default.
|
40
|
+
#
|
41
|
+
# Note: It can be overriden by `ONLINE_MIGRATIONS_VERBOSE_SQL_LOGS` environment variable.
|
42
|
+
config.verbose_sql_logs = defined?(Rails) && Rails.env.production?
|
43
|
+
|
35
44
|
# Lock retries.
|
36
45
|
# Configure your custom lock retrier (see LockRetrier).
|
37
46
|
# To disable lock retries, set `lock_retrier` to `nil`.
|
@@ -27,7 +27,7 @@ module OnlineMigrations
|
|
27
27
|
record.errors.add(
|
28
28
|
:migration_name,
|
29
29
|
"#{migration_name}#relation cannot use ORDER BY or LIMIT due to the way how iteration with a cursor is designed. " \
|
30
|
-
|
30
|
+
"You can use other ways to limit the number of rows, e.g. a WHERE condition on the primary key column."
|
31
31
|
)
|
32
32
|
end
|
33
33
|
end
|
@@ -101,7 +101,7 @@ module OnlineMigrations
|
|
101
101
|
|
102
102
|
if (extra_keys = (options.keys - conversions.keys)).any?
|
103
103
|
raise ArgumentError, "Options has unknown keys: #{extra_keys.map(&:inspect).join(', ')}. "\
|
104
|
-
|
104
|
+
"Can contain only column names: #{conversions.keys.map(&:inspect).join(', ')}."
|
105
105
|
end
|
106
106
|
|
107
107
|
transaction do
|
@@ -401,7 +401,7 @@ module OnlineMigrations
|
|
401
401
|
# This is necessary as we can't properly rename indexes such as "taggings_idx".
|
402
402
|
unless index.name.include?(from_column)
|
403
403
|
raise "The index #{index.name} can not be copied as it does not "\
|
404
|
-
|
404
|
+
"mention the old column. You have to rename this index manually first."
|
405
405
|
end
|
406
406
|
|
407
407
|
name = index.name.gsub(from_column, to_column)
|
@@ -19,11 +19,11 @@ module OnlineMigrations
|
|
19
19
|
end
|
20
20
|
|
21
21
|
def safety_assured
|
22
|
-
|
22
|
+
prev_value = @safe
|
23
23
|
@safe = true
|
24
24
|
yield
|
25
25
|
ensure
|
26
|
-
@safe =
|
26
|
+
@safe = prev_value
|
27
27
|
end
|
28
28
|
|
29
29
|
def check(command, *args, &block)
|
@@ -132,7 +132,7 @@ module OnlineMigrations
|
|
132
132
|
referenced_tables = foreign_keys.map(&:to_table).uniq
|
133
133
|
referenced_tables.delete(table_name.to_s) # ignore self references
|
134
134
|
|
135
|
-
if referenced_tables.
|
135
|
+
if referenced_tables.count { |t| !new_table?(t) } > 1
|
136
136
|
raise_error :drop_table_multiple_foreign_keys
|
137
137
|
end
|
138
138
|
end
|
@@ -155,9 +155,10 @@ module OnlineMigrations
|
|
155
155
|
end
|
156
156
|
|
157
157
|
def add_column(table_name, column_name, type, **options)
|
158
|
+
default = options[:default]
|
158
159
|
volatile_default = false
|
159
|
-
if !new_or_small_table?(table_name) && !
|
160
|
-
(postgresql_version < Gem::Version.new("11") || (volatile_default = Utils.volatile_default?(connection, type,
|
160
|
+
if !new_or_small_table?(table_name) && !default.nil? &&
|
161
|
+
(postgresql_version < Gem::Version.new("11") || (volatile_default = Utils.volatile_default?(connection, type, default)))
|
161
162
|
|
162
163
|
raise_error :add_column_with_default,
|
163
164
|
code: command_str(:add_column_with_default, table_name, column_name, type, options),
|
@@ -170,6 +171,20 @@ module OnlineMigrations
|
|
170
171
|
code: command_str(:add_column, table_name, column_name, :jsonb, options)
|
171
172
|
end
|
172
173
|
|
174
|
+
check_inheritance_column(table_name, column_name, default)
|
175
|
+
|
176
|
+
type = :bigint if type == :integer && options[:limit] == 8
|
177
|
+
check_mismatched_foreign_key_type(table_name, column_name, type)
|
178
|
+
end
|
179
|
+
|
180
|
+
def add_column_with_default(table_name, column_name, type, **options)
|
181
|
+
if type == :json
|
182
|
+
raise_error :add_column_json,
|
183
|
+
code: command_str(:add_column_with_default, table_name, column_name, :jsonb, options)
|
184
|
+
end
|
185
|
+
|
186
|
+
check_inheritance_column(table_name, column_name, options[:default])
|
187
|
+
|
173
188
|
type = :bigint if type == :integer && options[:limit] == 8
|
174
189
|
check_mismatched_foreign_key_type(table_name, column_name, type)
|
175
190
|
end
|
@@ -572,8 +587,12 @@ module OnlineMigrations
|
|
572
587
|
arg_list << last_arg.map do |k, v|
|
573
588
|
case v
|
574
589
|
when Hash
|
575
|
-
|
576
|
-
|
590
|
+
if v.empty?
|
591
|
+
"#{k}: {}"
|
592
|
+
else
|
593
|
+
# pretty index: { algorithm: :concurrently }
|
594
|
+
"#{k}: { #{v.map { |k2, v2| "#{k2}: #{v2.inspect}" }.join(', ')} }"
|
595
|
+
end
|
577
596
|
when Array, Numeric, String, Symbol, TrueClass, FalseClass
|
578
597
|
"#{k}: #{v.inspect}"
|
579
598
|
else
|
@@ -618,6 +637,14 @@ module OnlineMigrations
|
|
618
637
|
connection.select_all(constraints_query).to_a
|
619
638
|
end
|
620
639
|
|
640
|
+
def check_inheritance_column(table_name, column_name, default)
|
641
|
+
if column_name.to_s == ActiveRecord::Base.inheritance_column && !default.nil?
|
642
|
+
raise_error :add_inheritance_column,
|
643
|
+
table_name: table_name, column_name: column_name,
|
644
|
+
model: table_name.to_s.classify, subclass: default
|
645
|
+
end
|
646
|
+
end
|
647
|
+
|
621
648
|
def check_mismatched_foreign_key_type(table_name, column_name, type, **options)
|
622
649
|
column_name = column_name.to_s
|
623
650
|
ref_name = column_name.sub(/_id\z/, "")
|
@@ -144,6 +144,19 @@ module OnlineMigrations
|
|
144
144
|
#
|
145
145
|
attr_reader :enabled_checks
|
146
146
|
|
147
|
+
# Whether to log every SQL query happening in a migration
|
148
|
+
#
|
149
|
+
# This is useful to demystify online_migrations inner workings, and to better investigate
|
150
|
+
# migration failure in production. This is also useful in development to get
|
151
|
+
# a better grasp of what is going on for high-level statements like add_column_with_default.
|
152
|
+
#
|
153
|
+
# This feature is enabled by default in a production Rails environment.
|
154
|
+
# @return [Boolean]
|
155
|
+
#
|
156
|
+
# @note: It can be overriden by `ONLINE_MIGRATIONS_VERBOSE_SQL_LOGS` environment variable.
|
157
|
+
#
|
158
|
+
attr_accessor :verbose_sql_logs
|
159
|
+
|
147
160
|
# Configuration object to configure background migrations
|
148
161
|
#
|
149
162
|
# @return [BackgroundMigrationsConfig]
|
@@ -172,6 +185,7 @@ module OnlineMigrations
|
|
172
185
|
@small_tables = []
|
173
186
|
@check_down = false
|
174
187
|
@enabled_checks = @error_messages.keys.map { |k| [k, {}] }.to_h
|
188
|
+
@verbose_sql_logs = defined?(Rails) && Rails.env.production?
|
175
189
|
end
|
176
190
|
|
177
191
|
def lock_retrier=(value)
|
@@ -84,6 +84,31 @@ class <%= migration_name %> < <%= migration_parent %>
|
|
84
84
|
end
|
85
85
|
end",
|
86
86
|
|
87
|
+
add_inheritance_column:
|
88
|
+
"'<%= column_name %>' column is used for single table inheritance. Adding it might cause errors in old instances of your application.
|
89
|
+
|
90
|
+
After the migration was ran and the column was added, but before the code is fully deployed to all instances,
|
91
|
+
an old instance may be restarted (due to an error etc). And when it will fetch '<%= model %>' records from the database,
|
92
|
+
'<%= model %>' will look for a '<%= subclass %>' subclass (from the '<%= column_name %>' column) and fail to locate it unless it is already defined.
|
93
|
+
|
94
|
+
A safer approach is to:
|
95
|
+
|
96
|
+
1. ignore the column:
|
97
|
+
|
98
|
+
class <%= model %> < <%= model_parent %>
|
99
|
+
<% if ar_version >= 5 %>
|
100
|
+
self.ignored_columns = [\"<%= column_name %>\"]
|
101
|
+
<% else %>
|
102
|
+
def self.columns
|
103
|
+
super.reject { |c| c.name == \"<%= column_name %>\" }
|
104
|
+
end
|
105
|
+
<% end %>
|
106
|
+
end
|
107
|
+
|
108
|
+
2. deploy
|
109
|
+
3. remove the column ignoring from step 1 and apply initial code changes
|
110
|
+
4. deploy",
|
111
|
+
|
87
112
|
rename_column:
|
88
113
|
"Renaming a column that's in use will cause errors in your application.
|
89
114
|
migration_helpers provides a safer approach to do this:
|
@@ -225,7 +225,7 @@ module OnlineMigrations
|
|
225
225
|
# @see https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/
|
226
226
|
#
|
227
227
|
def delay(attempt)
|
228
|
-
(rand * [@max_delay, @base_delay * 2**(attempt - 1)].min).ceil
|
228
|
+
(rand * [@max_delay, @base_delay * (2**(attempt - 1))].min).ceil
|
229
229
|
end
|
230
230
|
end
|
231
231
|
|
@@ -4,24 +4,27 @@ module OnlineMigrations
|
|
4
4
|
module Migration
|
5
5
|
# @private
|
6
6
|
def migrate(direction)
|
7
|
+
VerboseSqlLogs.enable if verbose_sql_logs?
|
8
|
+
|
7
9
|
OnlineMigrations.current_migration = self
|
8
10
|
command_checker.direction = direction
|
11
|
+
|
9
12
|
super
|
13
|
+
ensure
|
14
|
+
VerboseSqlLogs.disable if verbose_sql_logs?
|
10
15
|
end
|
11
16
|
|
12
17
|
# @private
|
13
18
|
def method_missing(method, *args, &block)
|
14
|
-
if
|
19
|
+
if ar_schema?
|
15
20
|
super
|
16
21
|
elsif command_checker.check(method, *args, &block)
|
17
|
-
if
|
18
|
-
if method == :with_lock_retries
|
19
|
-
connection.with_lock_retries(*args, &block)
|
20
|
-
else
|
21
|
-
connection.with_lock_retries { super }
|
22
|
-
end
|
23
|
-
else
|
22
|
+
if in_transaction?
|
24
23
|
super
|
24
|
+
elsif method == :with_lock_retries
|
25
|
+
connection.with_lock_retries(*args, &block)
|
26
|
+
else
|
27
|
+
connection.with_lock_retries { super }
|
25
28
|
end
|
26
29
|
end
|
27
30
|
end
|
@@ -52,6 +55,19 @@ module OnlineMigrations
|
|
52
55
|
end
|
53
56
|
|
54
57
|
private
|
58
|
+
def verbose_sql_logs?
|
59
|
+
if (verbose = ENV["ONLINE_MIGRATIONS_VERBOSE_SQL_LOGS"])
|
60
|
+
Utils.to_bool(verbose)
|
61
|
+
else
|
62
|
+
OnlineMigrations.config.verbose_sql_logs
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def ar_schema?
|
67
|
+
is_a?(ActiveRecord::Schema) ||
|
68
|
+
(defined?(ActiveRecord::Schema::Definition) && is_a?(ActiveRecord::Schema::Definition))
|
69
|
+
end
|
70
|
+
|
55
71
|
def command_checker
|
56
72
|
@command_checker ||= CommandChecker.new(self)
|
57
73
|
end
|
@@ -84,6 +84,7 @@ module OnlineMigrations
|
|
84
84
|
model = Utils.define_model(self, table_name)
|
85
85
|
|
86
86
|
conditions = columns_and_values.map do |(column_name, value)|
|
87
|
+
value = Arel.sql(value.call.to_s) if value.is_a?(Proc)
|
87
88
|
arel_column = model.arel_table[column_name]
|
88
89
|
arel_column.not_eq(value).or(arel_column.eq(nil))
|
89
90
|
end
|
@@ -96,20 +97,28 @@ module OnlineMigrations
|
|
96
97
|
updates =
|
97
98
|
if Utils.ar_version <= 5.2
|
98
99
|
columns_and_values.map do |(column_name, value)|
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
100
|
+
rhs =
|
101
|
+
# ActiveRecord <= 5.2 can't quote these - we need to handle these cases manually
|
102
|
+
case value
|
103
|
+
when Arel::Attributes::Attribute
|
104
|
+
quote_column_name(value.name)
|
105
|
+
when Arel::Nodes::SqlLiteral
|
106
|
+
value
|
107
|
+
when Arel::Nodes::NamedFunction
|
108
|
+
"#{value.name}(#{quote_column_name(value.expressions.first.name)})"
|
109
|
+
when Proc
|
110
|
+
value.call
|
111
|
+
else
|
112
|
+
quote(value)
|
113
|
+
end
|
114
|
+
|
115
|
+
"#{quote_column_name(column_name)} = #{rhs}"
|
110
116
|
end.join(", ")
|
111
117
|
else
|
112
|
-
columns_and_values.
|
118
|
+
columns_and_values.map do |(column, value)|
|
119
|
+
value = Arel.sql(value.call.to_s) if value.is_a?(Proc)
|
120
|
+
[column, value]
|
121
|
+
end.to_h
|
113
122
|
end
|
114
123
|
|
115
124
|
relation.update_all(updates)
|
@@ -383,8 +392,7 @@ module OnlineMigrations
|
|
383
392
|
#
|
384
393
|
def add_column_with_default(table_name, column_name, type, **options)
|
385
394
|
default = options.fetch(:default)
|
386
|
-
if default.is_a?(Proc) &&
|
387
|
-
ActiveRecord.version < Gem::Version.new("5.0.0.beta2") # https://github.com/rails/rails/pull/20005
|
395
|
+
if default.is_a?(Proc) && Utils.ar_version < 5.0 # https://github.com/rails/rails/pull/20005
|
388
396
|
raise ArgumentError, "Expressions as default are not supported"
|
389
397
|
end
|
390
398
|
|
@@ -397,7 +405,7 @@ module OnlineMigrations
|
|
397
405
|
|
398
406
|
if column_exists?(table_name, column_name)
|
399
407
|
Utils.say("Column was not created because it already exists (this may be due to an aborted migration "\
|
400
|
-
|
408
|
+
"or similar) table_name: #{table_name}, column_name: #{column_name}")
|
401
409
|
else
|
402
410
|
transaction do
|
403
411
|
add_column(table_name, column_name, type, **options.merge(default: nil, null: true))
|
@@ -648,7 +656,7 @@ module OnlineMigrations
|
|
648
656
|
|
649
657
|
if __index_valid?(index_name, schema: schema)
|
650
658
|
Utils.say("Index was not created because it already exists (this may be due to an aborted migration "\
|
651
|
-
|
659
|
+
"or similar): table_name: #{table_name}, column_name: #{column_name}")
|
652
660
|
return
|
653
661
|
else
|
654
662
|
Utils.say("Recreating invalid index: table_name: #{table_name}, column_name: #{column_name}")
|
@@ -692,7 +700,7 @@ module OnlineMigrations
|
|
692
700
|
end
|
693
701
|
else
|
694
702
|
Utils.say("Index was not removed because it does not exist (this may be due to an aborted migration "\
|
695
|
-
|
703
|
+
"or similar): table_name: #{table_name}, column_name: #{column_names}")
|
696
704
|
end
|
697
705
|
end
|
698
706
|
|
@@ -703,7 +711,7 @@ module OnlineMigrations
|
|
703
711
|
def add_foreign_key(from_table, to_table, validate: true, **options)
|
704
712
|
if foreign_key_exists?(from_table, to_table, **options)
|
705
713
|
message = "Foreign key was not created because it already exists " \
|
706
|
-
|
714
|
+
"(this can be due to an aborted migration or similar): from_table: #{from_table}, to_table: #{to_table}".dup
|
707
715
|
message << ", #{options.inspect}" if options.any?
|
708
716
|
|
709
717
|
Utils.say(message)
|
@@ -761,7 +769,7 @@ module OnlineMigrations
|
|
761
769
|
|
762
770
|
if __check_constraint_exists?(table_name, constraint_name)
|
763
771
|
Utils.say("Check constraint was not created because it already exists (this may be due to an aborted migration "\
|
764
|
-
|
772
|
+
"or similar) table_name: #{table_name}, expression: #{expression}, constraint name: #{constraint_name}")
|
765
773
|
else
|
766
774
|
query = "ALTER TABLE #{table_name} ADD CONSTRAINT #{constraint_name} CHECK (#{expression})"
|
767
775
|
query += " NOT VALID" if !validate
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module OnlineMigrations
|
4
|
+
# @private
|
5
|
+
module VerboseSqlLogs
|
6
|
+
class << self
|
7
|
+
def enable
|
8
|
+
@activerecord_logger_was = ActiveRecord::Base.logger
|
9
|
+
@verbose_query_logs_was = verbose_query_logs
|
10
|
+
|
11
|
+
stdout_logger = ActiveSupport::Logger.new($stdout)
|
12
|
+
stdout_logger.formatter = @activerecord_logger_was.formatter
|
13
|
+
stdout_logger.level = @activerecord_logger_was.level
|
14
|
+
stdout_logger = ActiveSupport::TaggedLogging.new(stdout_logger)
|
15
|
+
|
16
|
+
combined_logger = stdout_logger.extend(ActiveSupport::Logger.broadcast(@activerecord_logger_was))
|
17
|
+
|
18
|
+
ActiveRecord::Base.logger = combined_logger
|
19
|
+
set_verbose_query_logs(false)
|
20
|
+
end
|
21
|
+
|
22
|
+
def disable
|
23
|
+
ActiveRecord::Base.logger = @activerecord_logger_was
|
24
|
+
set_verbose_query_logs(@verbose_query_logs_was)
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
def verbose_query_logs
|
29
|
+
if Utils.ar_version > 7.0
|
30
|
+
ActiveRecord.verbose_query_logs
|
31
|
+
elsif Utils.ar_version >= 5.2
|
32
|
+
ActiveRecord::Base.verbose_query_logs
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def set_verbose_query_logs(value) # rubocop:disable Naming/AccessorMethodName
|
37
|
+
if Utils.ar_version > 7.0
|
38
|
+
ActiveRecord.verbose_query_logs = value
|
39
|
+
elsif Utils.ar_version >= 5.2
|
40
|
+
ActiveRecord::Base.verbose_query_logs = value
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
data/lib/online_migrations.rb
CHANGED
@@ -6,6 +6,7 @@ require "online_migrations/utils"
|
|
6
6
|
require "online_migrations/error_messages"
|
7
7
|
require "online_migrations/config"
|
8
8
|
require "online_migrations/batch_iterator"
|
9
|
+
require "online_migrations/verbose_sql_logs"
|
9
10
|
require "online_migrations/migration"
|
10
11
|
require "online_migrations/migrator"
|
11
12
|
require "online_migrations/database_tasks"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: online_migrations
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- fatkodima
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-
|
11
|
+
date: 2022-02-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -88,6 +88,7 @@ files:
|
|
88
88
|
- lib/online_migrations/schema_cache.rb
|
89
89
|
- lib/online_migrations/schema_statements.rb
|
90
90
|
- lib/online_migrations/utils.rb
|
91
|
+
- lib/online_migrations/verbose_sql_logs.rb
|
91
92
|
- lib/online_migrations/version.rb
|
92
93
|
- online_migrations.gemspec
|
93
94
|
homepage: https://github.com/fatkodima/online_migrations
|